room_layer_manager.h

Go to the documentation of this file.

#ifndef YAZE_ZELDA3_DUNGEON_ROOM_LAYER_MANAGER_H
#define YAZE_ZELDA3_DUNGEON_ROOM_LAYER_MANAGER_H
 
#include <array>
#include <cstdint>
#include <vector>
 
#include "app/gfx/core/bitmap.h"
#include "app/gfx/render/background_buffer.h"
#include "zelda3/dungeon/room.h"
 
namespace yaze {
namespace zelda3 {
 
enum class LayerType {
  BG1_Layout,   // Base BG1 tiles from room layout
  BG1_Objects,  // Objects drawn to BG1 (Layer 0, 2)
  BG2_Layout,   // Base BG2 tiles from room layout
  BG2_Objects   // Objects drawn to BG2 (Layer 1)
};
 
enum class LayerBlendMode {
  Normal,       // Standard alpha blending
  Translucent,  // 50% alpha
  Addition,     // Additive blending
  Dark,         // Darkened blend
  Off           // Layer hidden
};
 
struct ObjectTranslucency {
  size_t object_index = 0;
  bool translucent = false;
  uint8_t alpha = 255;  // 0-255 alpha value
};
 
struct ObjectPriority {
  size_t object_index = 0;
  int layer = 0;               // Object's layer (0=BG1, 1=BG2, 2=BG1 priority)
  int priority = 0;            // Object layer value (not visual Z-order)
  bool is_bg2_object = false;  // True if object renders to BG2 buffer
};
 
class RoomLayerManager {
 public:
  RoomLayerManager() { Reset(); }
 
  // Reset to default state (all layers visible, normal blend)
  void Reset() {
    for (int i = 0; i < 4; ++i) {
      layer_visible_[i] = true;
      layer_blend_mode_[i] = LayerBlendMode::Normal;
      layer_alpha_[i] = 255;
    }
    object_translucency_.clear();
    bg2_on_top_ = false;
    game_hides_lower_tilemap_ = false;
    game_registers_applied_ = false;
    lower_tilemap_on_main_ = false;
    upper_tilemap_blended_ = false;
    cgadsub_ = 0;
    show_hidden_layers_ = false;
    layers_merged_ = false;
    current_merge_type_id_ = 0;
    use_priority_compositing_ =
        true;  // Default to SNES-informed priority order
  }
 
  // Priority compositing control
  // When enabled (default): Uses SNES Mode 1 per-tile priority for Z-ordering
  // When disabled: Simple back-to-front layer order (BG2 behind, BG1 in front)
  void SetPriorityCompositing(bool enabled) {
    use_priority_compositing_ = enabled;
  }
  bool IsPriorityCompositingEnabled() const {
    return use_priority_compositing_;
  }
 
  // The game's own layer settings for the room (room_layer_registers.h).
  // When the game shows neither tilemap bit 0 on main nor sub screen, the
  // lower tilemap (yaze's BG2 buffers) is not on screen in game, and the
  // composite hides it unless ShowHiddenLayers is on.
  void ApplyGameLayerRegisters(const RoomLayerRegisters& registers) {
    game_registers_applied_ = true;
    game_hides_lower_tilemap_ = !registers.LowerTilemapShown();
    lower_tilemap_on_main_ = registers.TilemapsShareMainScreen();
    upper_tilemap_blended_ = registers.UpperTilemapBlended();
    cgadsub_ = registers.cgadsub;
  }
  // Opaque upper-tilemap pixels always cover the lower tilemap: it is only on
  // the sub screen and the upper tilemap does not blend with it.
  bool UpperTilemapCoversLower(int layer2_mode) const {
    if (!game_registers_applied_) {
      return layer2_mode == 0x06;
    }
    return !lower_tilemap_on_main_ && !upper_tilemap_blended_;
  }
  // Both tilemaps are on the main screen; the lower one (hardware BG1) wins
  // ties at equal tile priority.
  bool LowerTilemapWinsTies() const {
    return game_registers_applied_ && lower_tilemap_on_main_;
  }
  bool GameHidesLowerTilemap() const { return game_hides_lower_tilemap_; }
  // Editing aid: draw layers the game hides.
  void SetShowHiddenLayers(bool show) { show_hidden_layers_ = show; }
  bool ShowHiddenLayers() const { return show_hidden_layers_; }
  // True when `layer` is hidden only because the game hides it.
  bool IsHiddenByGame(LayerType layer) const {
    return game_hides_lower_tilemap_ && !show_hidden_layers_ &&
           (layer == LayerType::BG2_Layout || layer == LayerType::BG2_Objects);
  }
 
  // Layer visibility
  void SetLayerVisible(LayerType layer, bool visible) {
    layer_visible_[static_cast<int>(layer)] = visible;
  }
 
  bool IsLayerVisible(LayerType layer) const {
    return layer_visible_[static_cast<int>(layer)];
  }
 
  // Layer blend mode
  void SetLayerBlendMode(LayerType layer, LayerBlendMode mode) {
    layer_blend_mode_[static_cast<int>(layer)] = mode;
    // Update alpha based on blend mode
    switch (mode) {
      case LayerBlendMode::Normal:
        layer_alpha_[static_cast<int>(layer)] = 255;
        break;
      case LayerBlendMode::Translucent:
        layer_alpha_[static_cast<int>(layer)] = 180;
        break;
      case LayerBlendMode::Addition:
        layer_alpha_[static_cast<int>(layer)] = 220;
        break;
      case LayerBlendMode::Dark:
        layer_alpha_[static_cast<int>(layer)] = 120;
        break;
      case LayerBlendMode::Off:
        layer_alpha_[static_cast<int>(layer)] = 0;
        break;
    }
  }
 
  LayerBlendMode GetLayerBlendMode(LayerType layer) const {
    return layer_blend_mode_[static_cast<int>(layer)];
  }
 
  uint8_t GetLayerAlpha(LayerType layer) const {
    return layer_alpha_[static_cast<int>(layer)];
  }
 
  // Per-object translucency
  void SetObjectTranslucency(size_t object_index, bool translucent,
                             uint8_t alpha = 128) {
    // Find existing entry or add new one
    for (auto& entry : object_translucency_) {
      if (entry.object_index == object_index) {
        entry.translucent = translucent;
        entry.alpha = alpha;
        return;
      }
    }
    object_translucency_.push_back({object_index, translucent, alpha});
  }
 
  bool IsObjectTranslucent(size_t object_index) const {
    for (const auto& entry : object_translucency_) {
      if (entry.object_index == object_index) {
        return entry.translucent;
      }
    }
    return false;
  }
 
  uint8_t GetObjectAlpha(size_t object_index) const {
    for (const auto& entry : object_translucency_) {
      if (entry.object_index == object_index && entry.translucent) {
        return entry.alpha;
      }
    }
    return 255;
  }
 
  void ClearObjectTranslucency() { object_translucency_.clear(); }
 
  // Color math participation flag (from LayerMergeType.Layer2OnTop)
  // NOTE: This does NOT affect draw order - BG1 is always above BG2.
  // This flag controls whether BG2 participates in sub-screen color math
  // effects like transparency and additive blending.
  void SetBG2ColorMathEnabled(bool enabled) { bg2_on_top_ = enabled; }
  bool IsBG2ColorMathEnabled() const { return bg2_on_top_; }
 
  // Legacy aliases for compatibility
  void SetBG2OnTop(bool on_top) { bg2_on_top_ = on_top; }
  bool IsBG2OnTop() const { return bg2_on_top_; }
 
  // Apply layer settings to room from LayerMergeType
  // NOTE: This affects BLEND MODES and COLOR MATH, not draw order.
  // SNES Mode 1 always renders BG1 above BG2 - this is hardware behavior.
  // Layer visibility checkboxes remain independent of merge type.
  void ApplyLayerMerging(const LayerMergeType& merge_type) {
    // Store the current merge type for queries
    current_merge_type_id_ = merge_type.ID;
    layers_merged_ = (merge_type.ID != 0);  // ID 0 = "Off" = not merged
 
    // Set BG2 color math participation (does NOT change draw order)
    SetBG2ColorMathEnabled(merge_type.Layer2OnTop);
 
    // Apply blend mode based on merge type
    // NOTE: Layer2Visible from ROM is informational only - user can still
    // enable/disable layers via checkboxes. We only set blend modes here.
    // Layer2Translucent = true means BG2 should use translucent blend
    if (merge_type.Layer2Translucent) {
      SetLayerBlendMode(LayerType::BG2_Layout, LayerBlendMode::Translucent);
      SetLayerBlendMode(LayerType::BG2_Objects, LayerBlendMode::Translucent);
    } else {
      SetLayerBlendMode(LayerType::BG2_Layout, LayerBlendMode::Normal);
      SetLayerBlendMode(LayerType::BG2_Objects, LayerBlendMode::Normal);
    }
 
    // BG1 blend mode depends on merge type
    // DarkRoom (ID 0x08) should darken BG1 to simulate unlit room
    if (merge_type.ID == 0x08) {
      // Dark room - BG1 is dimmed (reduced brightness)
      SetLayerBlendMode(LayerType::BG1_Layout, LayerBlendMode::Dark);
      SetLayerBlendMode(LayerType::BG1_Objects, LayerBlendMode::Dark);
    } else {
      SetLayerBlendMode(LayerType::BG1_Layout, LayerBlendMode::Normal);
      SetLayerBlendMode(LayerType::BG1_Objects, LayerBlendMode::Normal);
    }
  }
 
  // Apply room effect-specific visual hints on top of merge settings.
  // This keeps the editor preview closer to in-game behavior for effect-driven
  // rooms (e.g. Moving Water) without changing layer visibility policy.
  void ApplyRoomEffect(EffectKey effect) {
    switch (effect) {
      case EffectKey::Moving_Water:
        // Water rooms: BG2 shows water with translucent overlay on BG1.
        // SNES uses HDMA-driven color math to blend water layer.
        if (GetLayerBlendMode(LayerType::BG2_Layout) ==
            LayerBlendMode::Normal) {
          SetLayerBlendMode(LayerType::BG2_Layout, LayerBlendMode::Translucent);
        }
        if (GetLayerBlendMode(LayerType::BG2_Objects) ==
            LayerBlendMode::Normal) {
          SetLayerBlendMode(LayerType::BG2_Objects,
                            LayerBlendMode::Translucent);
        }
        break;
 
      case EffectKey::Moving_Floor:
        // Conveyor belt rooms: BG2 scrolls independently.
        // No blend change needed - BG2 is opaque floor tiles.
        break;
 
      case EffectKey::Torch_Show_Floor:
        // Dark rooms where lighting a torch reveals BG2 floor.
        // BG1 is the dark overlay, BG2 is the revealed floor.
        // In-game, HDMA window controls which scanlines show BG2.
        // For editor preview: darken BG1, keep BG2 visible.
        SetLayerBlendMode(LayerType::BG1_Layout, LayerBlendMode::Dark);
        SetLayerBlendMode(LayerType::BG1_Objects, LayerBlendMode::Dark);
        break;
 
      case EffectKey::Red_Flashes:
        // Lightning/flash effect (Ganon fight). No persistent blend change.
        break;
 
      case EffectKey::Ganon_Room:
        // Ganon's room: special rendering with translucent BG2 for
        // the Triforce floor pattern showing through.
        if (GetLayerBlendMode(LayerType::BG2_Layout) ==
            LayerBlendMode::Normal) {
          SetLayerBlendMode(LayerType::BG2_Layout, LayerBlendMode::Translucent);
        }
        break;
 
      default:
        break;
    }
  }
 
  void ApplyLayerMergingPreserveVisibility(const LayerMergeType& merge_type) {
    // Store visibility before applying
    bool bg1_layout_vis = IsLayerVisible(LayerType::BG1_Layout);
    bool bg1_objects_vis = IsLayerVisible(LayerType::BG1_Objects);
    bool bg2_layout_vis = IsLayerVisible(LayerType::BG2_Layout);
    bool bg2_objects_vis = IsLayerVisible(LayerType::BG2_Objects);
 
    // Apply merge settings
    ApplyLayerMerging(merge_type);
 
    // Restore visibility
    SetLayerVisible(LayerType::BG1_Layout, bg1_layout_vis);
    SetLayerVisible(LayerType::BG1_Objects, bg1_objects_vis);
    SetLayerVisible(LayerType::BG2_Layout, bg2_layout_vis);
    SetLayerVisible(LayerType::BG2_Objects, bg2_objects_vis);
  }
 
  bool AreLayersMerged() const { return layers_merged_; }
 
  uint8_t GetMergeTypeId() const { return current_merge_type_id_; }
 
  // Signature for caching composited output. If any layer/blend/translucency
  // state changes, callers should treat previously cached composite bitmaps as
  // stale.
  uint64_t CompositeStateSignature() const {
    uint64_t sig = 1469598103934665603ull;
    const auto mix = [&sig](uint64_t value) {
      sig ^= value;
      sig *= 1099511628211ull;
    };
 
    for (int i = 0; i < 4; ++i) {
      mix(layer_visible_[i] ? 1u : 0u);
      mix(static_cast<uint64_t>(layer_blend_mode_[i]));
      mix(layer_alpha_[i]);
    }
 
    mix(bg2_on_top_ ? 1u : 0u);
    mix(game_hides_lower_tilemap_ ? 1u : 0u);
    mix(game_registers_applied_ ? 1u : 0u);
    mix(lower_tilemap_on_main_ ? 1u : 0u);
    mix(upper_tilemap_blended_ ? 1u : 0u);
    mix(cgadsub_);
    mix(show_hidden_layers_ ? 1u : 0u);
    mix(layers_merged_ ? 1u : 0u);
    mix(current_merge_type_id_);
    mix(use_priority_compositing_ ? 1u : 0u);
 
    mix(static_cast<uint64_t>(object_translucency_.size()));
    for (const auto& entry : object_translucency_) {
      mix(static_cast<uint64_t>(entry.object_index));
      mix(entry.translucent ? 1u : 0u);
      mix(entry.alpha);
    }
 
    return sig;
  }
 
  // ============================================================================
  // Object Layer Assignment
  // ============================================================================
 
  int GetObjectLayerValue(int object_layer) const { return object_layer; }
 
  // Legacy function - kept for API compatibility
  // No longer affects visual order since SNES Mode 1 is fixed (BG1 > BG2)
  int CalculateObjectPriority(int object_layer) const {
    return object_layer * 10;
  }
 
  static bool IsObjectOnBG2(int object_layer) {
    // Layer 1 objects render to BG2, layers 0 and 2 render to BG1
    return object_layer == 1;
  }
 
  static LayerType GetObjectLayerType(int object_layer) {
    return IsObjectOnBG2(object_layer) ? LayerType::BG2_Objects
                                       : LayerType::BG1_Objects;
  }
 
  std::array<LayerType, 4> GetDrawOrder() const {
    // Standard SNES Mode 1 order: BG2 behind BG1
    // bg2_on_top_ affects blend modes, not draw order
    return {LayerType::BG2_Layout, LayerType::BG2_Objects,
            LayerType::BG1_Layout, LayerType::BG1_Objects};
  }
 
  static gfx::BackgroundBuffer& GetLayerBuffer(Room& room, LayerType layer) {
    switch (layer) {
      case LayerType::BG1_Layout:
        return room.bg1_buffer();
      case LayerType::BG1_Objects:
        return room.object_bg1_buffer();
      case LayerType::BG2_Layout:
        return room.bg2_buffer();
      case LayerType::BG2_Objects:
        return room.object_bg2_buffer();
    }
    // Fallback (should never reach)
    return room.bg1_buffer();
  }
 
  static const gfx::BackgroundBuffer& GetLayerBuffer(const Room& room,
                                                     LayerType layer) {
    switch (layer) {
      case LayerType::BG1_Layout:
        return room.bg1_buffer();
      case LayerType::BG1_Objects:
        return room.object_bg1_buffer();
      case LayerType::BG2_Layout:
        return room.bg2_buffer();
      case LayerType::BG2_Objects:
        return room.object_bg2_buffer();
    }
    // Fallback (should never reach)
    return room.bg1_buffer();
  }
 
  static const char* GetLayerName(LayerType layer) {
    switch (layer) {
      case LayerType::BG1_Layout:
        return "BG1 Layout";
      case LayerType::BG1_Objects:
        return "BG1 Objects";
      case LayerType::BG2_Layout:
        return "BG2 Layout";
      case LayerType::BG2_Objects:
        return "BG2 Objects";
    }
    return "Unknown";
  }
 
  static const char* GetBlendModeName(LayerBlendMode mode) {
    switch (mode) {
      case LayerBlendMode::Normal:
        return "Normal";
      case LayerBlendMode::Translucent:
        return "Translucent";
      case LayerBlendMode::Addition:
        return "Addition";
      case LayerBlendMode::Dark:
        return "Dark";
      case LayerBlendMode::Off:
        return "Off";
    }
    return "Unknown";
  }
 
  void ApplySurfaceColorMod(SDL_Surface* surface) const {
    if (!surface)
      return;
 
    if (current_merge_type_id_ == 0x08) {
      // DarkRoom: 50% brightness
      SDL_SetSurfaceColorMod(surface, 128, 128, 128);
    } else {
      // Normal: Full brightness
      SDL_SetSurfaceColorMod(surface, 255, 255, 255);
    }
  }
 
  // ============================================================================
  // Layer Compositing
  // ============================================================================
 
  void CompositeToOutput(Room& room, gfx::Bitmap& output) const;
 
 private:
  static bool IsTransparent(uint8_t pixel) { return pixel == 255; }
 
  std::array<bool, 4> layer_visible_;
  std::array<LayerBlendMode, 4> layer_blend_mode_;
  std::array<uint8_t, 4> layer_alpha_;
  std::vector<ObjectTranslucency> object_translucency_;
 
  // Color math participation flag (from ROM's Layer2OnTop)
  // NOTE: Does NOT affect draw order - BG1 is always above BG2 per SNES Mode 1.
  // This controls whether BG2 participates in sub-screen color math effects.
  bool bg2_on_top_ = false;
  bool game_hides_lower_tilemap_ = false;
  bool game_registers_applied_ = false;
  bool lower_tilemap_on_main_ = false;
  bool upper_tilemap_blended_ = false;
  uint8_t cgadsub_ = 0;
  bool show_hidden_layers_ = false;
 
  // Merge state tracking
  bool layers_merged_ = false;
  uint8_t current_merge_type_id_ = 0;
 
  // When enabled, CompositeToOutput uses per-pixel priority buffers to emulate
  // SNES Mode 1 ordering (BG2 pri=1 can appear above BG1 pri=0).
  bool use_priority_compositing_ = true;
};
 
}  // namespace zelda3
}  // namespace yaze
 
#endif  // YAZE_ZELDA3_DUNGEON_ROOM_LAYER_MANAGER_H