dds.cpp

// Minimal DDS reader/writer, see dds.h. A .dds file is a 128-byte header (plus 20 more for the DX10
// flavour) followed by the mip levels, largest first, each one tightly packed. D3D on the other hand
// hands out locked surfaces with a pitch that can be wider than the row, so everything in here
// copies row by row.
#include "dds.h"
#include <cstdio>
#include <cstring>
#include <vector>

namespace {

#pragma pack(push, 1)
struct DdsPixelFormat { unsigned size, flags, fourCC, rgbBitCount, rMask, gMask, bMask, aMask; };
struct DdsHeader {
    unsigned magic, size, flags, height, width, pitchOrLinearSize, depth, mipMapCount, reserved1[11];
    DdsPixelFormat pf;
    unsigned caps, caps2, caps3, caps4, reserved2;
};
struct DdsHeaderDx10 { unsigned dxgiFormat, resourceDimension, miscFlag, arraySize, miscFlags2; };
#pragma pack(pop)

constexpr unsigned kMagic = 0x20534444;   // "DDS "
constexpr unsigned DDPF_ALPHAPIXELS = 0x1, DDPF_ALPHA = 0x2, DDPF_FOURCC = 0x4, DDPF_RGB = 0x40, DDPF_LUMINANCE = 0x20000;
constexpr unsigned DDSD_CAPS = 0x1, DDSD_HEIGHT = 0x2, DDSD_WIDTH = 0x4, DDSD_PITCH = 0x8, DDSD_PIXELFORMAT = 0x1000, DDSD_MIPMAPCOUNT = 0x20000, DDSD_LINEARSIZE = 0x80000;
constexpr unsigned DDSCAPS_COMPLEX = 0x8, DDSCAPS_TEXTURE = 0x1000, DDSCAPS_MIPMAP = 0x400000;
constexpr unsigned DDSCAPS2_CUBEMAP = 0x200, DDSCAPS2_VOLUME = 0x200000;
constexpr unsigned FourCC(char a, char b, char c, char d) { return unsigned(a) | unsigned(b) << 8 | unsigned(c) << 16 | unsigned(d) << 24; }
constexpr unsigned kDXT1 = FourCC('D', 'X', 'T', '1'), kDXT2 = FourCC('D', 'X', 'T', '2'), kDXT3 = FourCC('D', 'X', 'T', '3');
constexpr unsigned kDXT4 = FourCC('D', 'X', 'T', '4'), kDXT5 = FourCC('D', 'X', 'T', '5'), kDX10 = FourCC('D', 'X', '1', '0');

void SetErr(char* err, int len, const char* msg) { if (err && len > 0) { strncpy(err, msg, len - 1); err[len - 1] = 0; } }

bool IsCompressed(D3DFORMAT f) { return f == D3DFMT_DXT1 || f == D3DFMT_DXT2 || f == D3DFMT_DXT3 || f == D3DFMT_DXT4 || f == D3DFMT_DXT5; }
int BytesPerPixel(D3DFORMAT f)
{
    switch (f) {
    case D3DFMT_A8R8G8B8: case D3DFMT_X8R8G8B8: case D3DFMT_A8B8G8R8: case D3DFMT_X8B8G8R8: return 4;
    case D3DFMT_R8G8B8: return 3;
    case D3DFMT_R5G6B5: case D3DFMT_A1R5G5B5: case D3DFMT_X1R5G5B5: case D3DFMT_A4R4G4B4: case D3DFMT_A8L8: return 2;
    case D3DFMT_L8: case D3DFMT_A8: case D3DFMT_P8: return 1;
    default: return 0;
    }
}
// Size in bytes of one mip level as stored in a DDS file (tightly packed). For the DXT formats a
// "row" is a row of 4x4 blocks: 8 bytes per block for DXT1, 16 for the others.
unsigned LevelSize(D3DFORMAT f, unsigned w, unsigned h, unsigned* rowBytes, unsigned* rows)
{
    if (IsCompressed(f)) {
        unsigned bw = (w + 3) / 4, bh = (h + 3) / 4, bs = (f == D3DFMT_DXT1) ? 8 : 16;
        *rowBytes = bw * bs; *rows = bh; return bw * bh * bs;
    }
    *rowBytes = w * BytesPerPixel(f); *rows = h; return *rowBytes * h;
}

// Maps the header of a file to the D3D9 format to create. Photoshop, GIMP, Paint.NET and texconv do
// not agree on how to write a header, so this is a bit more forgiving than the spec: newer tools
// write BC1-3 with a DX10 header, which is the same data as DXT1/3/5.
D3DFORMAT FormatFromHeader(const DdsHeader& h, const DdsHeaderDx10* dx10)
{
    const DdsPixelFormat& pf = h.pf;
    if (pf.flags & DDPF_FOURCC) {
        if (pf.fourCC == kDXT1) return D3DFMT_DXT1;
        if (pf.fourCC == kDXT2) return D3DFMT_DXT2;
        if (pf.fourCC == kDXT3) return D3DFMT_DXT3;
        if (pf.fourCC == kDXT4) return D3DFMT_DXT4;
        if (pf.fourCC == kDXT5) return D3DFMT_DXT5;
        if (pf.fourCC == kDX10 && dx10) {
            switch (dx10->dxgiFormat) {
            case 70: case 71: case 72: return D3DFMT_DXT1;   // BC1
            case 73: case 74: case 75: return D3DFMT_DXT3;   // BC2
            case 76: case 77: case 78: return D3DFMT_DXT5;   // BC3
            case 87: case 88: return D3DFMT_A8R8G8B8;        // B8G8R8A8
            case 28: case 29: return D3DFMT_A8B8G8R8;        // R8G8B8A8
            default: return D3DFMT_UNKNOWN;
            }
        }
        return D3DFMT_UNKNOWN;
    }
    if (pf.flags & DDPF_RGB) {
        bool alpha = (pf.flags & DDPF_ALPHAPIXELS) && pf.aMask;
        switch (pf.rgbBitCount) {
        case 32:
            if (pf.rMask == 0x00FF0000) return alpha ? D3DFMT_A8R8G8B8 : D3DFMT_X8R8G8B8;
            if (pf.rMask == 0x000000FF) return alpha ? D3DFMT_A8B8G8R8 : D3DFMT_X8B8G8R8;
            return D3DFMT_UNKNOWN;
        case 24: return D3DFMT_R8G8B8;
        case 16:
            if (pf.rMask == 0xF800) return D3DFMT_R5G6B5;
            if (pf.rMask == 0x7C00) return alpha ? D3DFMT_A1R5G5B5 : D3DFMT_X1R5G5B5;
            if (pf.rMask == 0x0F00) return D3DFMT_A4R4G4B4;
            return D3DFMT_UNKNOWN;
        default: return D3DFMT_UNKNOWN;
        }
    }
    if (pf.flags & DDPF_LUMINANCE) {
        if (pf.rgbBitCount == 8) return D3DFMT_L8;
        if (pf.rgbBitCount == 16) return D3DFMT_A8L8;
    }
    if (pf.rgbBitCount == 8 && pf.aMask == 0xFF) return D3DFMT_A8;
    return D3DFMT_UNKNOWN;
}

// The other direction, for the export. False for formats a DDS header of this kind cannot describe (P8, depth, float).
bool PixelFormatFor(D3DFORMAT f, DdsPixelFormat& pf)
{
    pf = {}; pf.size = 32;
    auto rgb = [&](unsigned bits, unsigned r, unsigned g, unsigned b, unsigned a) {
        pf.flags = DDPF_RGB | (a ? DDPF_ALPHAPIXELS : 0); pf.rgbBitCount = bits; pf.rMask = r; pf.gMask = g; pf.bMask = b; pf.aMask = a; return true; };
    auto fourcc = [&](unsigned cc) { pf.flags = DDPF_FOURCC; pf.fourCC = cc; return true; };
    switch (f) {
    case D3DFMT_DXT1: return fourcc(kDXT1);
    case D3DFMT_DXT2: return fourcc(kDXT2);
    case D3DFMT_DXT3: return fourcc(kDXT3);
    case D3DFMT_DXT4: return fourcc(kDXT4);
    case D3DFMT_DXT5: return fourcc(kDXT5);
    case D3DFMT_A8R8G8B8: return rgb(32, 0x00FF0000, 0x0000FF00, 0x000000FF, 0xFF000000);
    case D3DFMT_X8R8G8B8: return rgb(32, 0x00FF0000, 0x0000FF00, 0x000000FF, 0);
    case D3DFMT_A8B8G8R8: return rgb(32, 0x000000FF, 0x0000FF00, 0x00FF0000, 0xFF000000);
    case D3DFMT_X8B8G8R8: return rgb(32, 0x000000FF, 0x0000FF00, 0x00FF0000, 0);
    case D3DFMT_R8G8B8:   return rgb(24, 0x00FF0000, 0x0000FF00, 0x000000FF, 0);
    case D3DFMT_R5G6B5:   return rgb(16, 0xF800, 0x07E0, 0x001F, 0);
    case D3DFMT_A1R5G5B5: return rgb(16, 0x7C00, 0x03E0, 0x001F, 0x8000);
    case D3DFMT_X1R5G5B5: return rgb(16, 0x7C00, 0x03E0, 0x001F, 0);
    case D3DFMT_A4R4G4B4: return rgb(16, 0x0F00, 0x00F0, 0x000F, 0xF000);
    case D3DFMT_L8:  pf.flags = DDPF_LUMINANCE; pf.rgbBitCount = 8;  pf.rMask = 0xFF; return true;
    case D3DFMT_A8L8: pf.flags = DDPF_LUMINANCE | DDPF_ALPHAPIXELS; pf.rgbBitCount = 16; pf.rMask = 0xFF; pf.aMask = 0xFF00; return true;
    case D3DFMT_A8:  pf.flags = DDPF_ALPHA; pf.rgbBitCount = 8; pf.aMask = 0xFF; return true;   // alpha only: DDPF_ALPHA, not ALPHAPIXELS
    default: return false;
    }
}

bool ReadFileBytes(const char* path, std::vector<unsigned char>& out)
{
    FILE* f = fopen(path, "rb");
    if (!f) return false;
    fseek(f, 0, SEEK_END); long n = ftell(f); fseek(f, 0, SEEK_SET);
    if (n <= 0) { fclose(f); return false; }
    out.resize(static_cast<size_t>(n));
    size_t got = fread(out.data(), 1, out.size(), f);
    fclose(f);
    return got == out.size();
}

// Appends one mip level to out, tightly packed. Render targets live in video memory and cannot be
// locked, so their level is copied to a system-memory surface first.
bool ReadLevel(IDirect3DTexture9* tex, unsigned level, std::vector<unsigned char>& out, char* err, int errLen)
{
    D3DSURFACE_DESC d;
    tex->GetLevelDesc(level, &d);
    unsigned rowBytes, rows;
    unsigned size = LevelSize(d.Format, d.Width, d.Height, &rowBytes, &rows);
    auto copy = [&](const D3DLOCKED_RECT& lr) {
        size_t at = out.size(); out.resize(at + size);
        for (unsigned r = 0; r < rows; ++r) memcpy(out.data() + at + r * rowBytes, static_cast<const unsigned char*>(lr.pBits) + r * lr.Pitch, rowBytes);
    };
    D3DLOCKED_RECT lr;
    if (SUCCEEDED(tex->LockRect(level, &lr, nullptr, D3DLOCK_READONLY))) { copy(lr); tex->UnlockRect(level); return true; }
    if (!(d.Usage & D3DUSAGE_RENDERTARGET)) { SetErr(err, errLen, "LockRect failed (texture not lockable)"); return false; }

    IDirect3DDevice9* dev = nullptr;
    IDirect3DSurface9* src = nullptr, *mem = nullptr;
    HRESULT hr = tex->GetDevice(&dev);
    if (SUCCEEDED(hr)) hr = tex->GetSurfaceLevel(level, &src);
    if (SUCCEEDED(hr)) hr = dev->CreateOffscreenPlainSurface(d.Width, d.Height, d.Format, D3DPOOL_SYSTEMMEM, &mem, nullptr);
    if (SUCCEEDED(hr)) hr = dev->GetRenderTargetData(src, mem);
    if (SUCCEEDED(hr)) hr = mem->LockRect(&lr, nullptr, D3DLOCK_READONLY);
    if (SUCCEEDED(hr)) { copy(lr); mem->UnlockRect(); }
    if (mem) mem->Release();
    if (src) src->Release();
    if (dev) dev->Release();
    if (FAILED(hr)) { char b[80]; snprintf(b, sizeof(b), "render target could not be read (0x%08X)", static_cast<unsigned>(hr)); SetErr(err, errLen, b); return false; }
    return true;
}

} // namespace

IDirect3DTexture9* DdsLoad(IDirect3DDevice9* dev, const char* path, char* err, int errLen)
{
    std::vector<unsigned char> bytes;
    if (!ReadFileBytes(path, bytes)) { SetErr(err, errLen, "cannot read file"); return nullptr; }
    if (bytes.size() < sizeof(DdsHeader)) { SetErr(err, errLen, "file too small"); return nullptr; }
    const DdsHeader* h = reinterpret_cast<const DdsHeader*>(bytes.data());
    if (h->magic != kMagic || h->size != 124) { SetErr(err, errLen, "not a DDS file"); return nullptr; }
    size_t dataOfs = sizeof(DdsHeader);
    const DdsHeaderDx10* dx10 = nullptr;
    if ((h->pf.flags & DDPF_FOURCC) && h->pf.fourCC == kDX10) {
        if (bytes.size() < dataOfs + sizeof(DdsHeaderDx10)) { SetErr(err, errLen, "truncated DX10 header"); return nullptr; }
        dx10 = reinterpret_cast<const DdsHeaderDx10*>(bytes.data() + dataOfs);
        dataOfs += sizeof(DdsHeaderDx10);
    }
    D3DFORMAT fmt = FormatFromHeader(*h, dx10);
    if (fmt == D3DFMT_UNKNOWN) { SetErr(err, errLen, "unsupported pixel format (use DXT1/3/5, BC1-3 or 32-bit RGBA)"); return nullptr; }
    unsigned w = h->width, hgt = h->height;
    if (!w || !hgt) { SetErr(err, errLen, "zero size"); return nullptr; }
    // These would load as a 2D texture made of their first face or slice. Better to say no.
    if (h->caps2 & (DDSCAPS2_CUBEMAP | DDSCAPS2_VOLUME)) { SetErr(err, errLen, "cube maps and volume textures are not supported"); return nullptr; }
    unsigned mips = (h->flags & DDSD_MIPMAPCOUNT) && h->mipMapCount ? h->mipMapCount : 1;
    if (IsCompressed(fmt) && ((w & 3) || (hgt & 3))) { SetErr(err, errLen, "DXT size must be a multiple of 4"); return nullptr; }

    // MANAGED, so the texture survives a device reset (alt-tab, resolution change) without our help.
    IDirect3DTexture9* tex = nullptr;
    HRESULT hr = dev->CreateTexture(w, hgt, mips, 0, fmt, D3DPOOL_MANAGED, &tex, nullptr);
    if (FAILED(hr) || !tex) { char b[96]; snprintf(b, sizeof(b), "CreateTexture failed (0x%08X)", static_cast<unsigned>(hr)); SetErr(err, errLen, b); return nullptr; }

    size_t ofs = dataOfs;
    unsigned lw = w, lh = hgt;
    for (unsigned level = 0; level < mips; ++level) {
        unsigned rowBytes, rows;
        unsigned size = LevelSize(fmt, lw, lh, &rowBytes, &rows);
        if (ofs + size > bytes.size()) { tex->Release(); SetErr(err, errLen, "file shorter than its mip chain"); return nullptr; }
        D3DLOCKED_RECT lr;
        if (FAILED(tex->LockRect(level, &lr, nullptr, 0))) { tex->Release(); SetErr(err, errLen, "LockRect failed"); return nullptr; }
        const unsigned char* src = bytes.data() + ofs;
        for (unsigned r = 0; r < rows; ++r) memcpy(static_cast<unsigned char*>(lr.pBits) + r * lr.Pitch, src + r * rowBytes, rowBytes);
        tex->UnlockRect(level);
        ofs += size;
        lw = lw > 1 ? lw / 2 : 1; lh = lh > 1 ? lh / 2 : 1;
    }
    return tex;
}

bool DdsSave(IDirect3DTexture9* tex, const char* path, char* err, int errLen)
{
    D3DSURFACE_DESC d0;
    if (!tex || FAILED(tex->GetLevelDesc(0, &d0))) { SetErr(err, errLen, "no texture"); return false; }
    DdsHeader h = {};
    if (!PixelFormatFor(d0.Format, h.pf)) { char b[64]; snprintf(b, sizeof(b), "format %d not exportable", d0.Format); SetErr(err, errLen, b); return false; }
    unsigned mips = tex->GetLevelCount();
    unsigned rowBytes, rows;
    unsigned size0 = LevelSize(d0.Format, d0.Width, d0.Height, &rowBytes, &rows);
    h.magic = kMagic; h.size = 124;
    h.flags = DDSD_CAPS | DDSD_HEIGHT | DDSD_WIDTH | DDSD_PIXELFORMAT | (IsCompressed(d0.Format) ? DDSD_LINEARSIZE : DDSD_PITCH) | (mips > 1 ? DDSD_MIPMAPCOUNT : 0);
    h.height = d0.Height; h.width = d0.Width;
    h.pitchOrLinearSize = IsCompressed(d0.Format) ? size0 : rowBytes;
    h.mipMapCount = mips;
    h.caps = DDSCAPS_TEXTURE | (mips > 1 ? (DDSCAPS_COMPLEX | DDSCAPS_MIPMAP) : 0);

    // Build the whole file in memory first, so a texture that cannot be read leaves no half-written file behind.
    std::vector<unsigned char> out(reinterpret_cast<unsigned char*>(&h), reinterpret_cast<unsigned char*>(&h) + sizeof(h));
    for (unsigned level = 0; level < mips; ++level)
        if (!ReadLevel(tex, level, out, err, errLen)) return false;
    FILE* f = fopen(path, "wb");
    if (!f) { SetErr(err, errLen, "cannot create file"); return false; }
    bool ok = fwrite(out.data(), 1, out.size(), f) == out.size();
    fclose(f);
    if (!ok) SetErr(err, errLen, "write failed");
    return ok;
}