Improve WangBrush and TileLayerWangEdit behavior with empty color

Mostly removing the special handling for erasing and changing the way filling is implemented.
mapeditor · bjorn · Jul 6, 2023 · Jul 4, 2023 · Jul 4, 2023 · Jul 5, 2023
commit 745addc8213a44e5220f75febf0cd30eab453df8
diff --git a/docs/manual/terrain.rst b/docs/manual/terrain.rst
@@ -49,7 +49,7 @@ Mixed Set
 Based on the information in a terrain set, the :ref:`terrain-tool` can
 understand the map and automatically choose the right tiles when making edits.
 When necessary, it also adjusts neighboring tiles to make sure they correctly
-connect to the modified area. A terrain set can contain up to 255 terrains.
+connect to the modified area. A terrain set can contain up to 254 terrains.
 
 The :ref:`stamp-tool`, as well as the :ref:`bucket-fill-tool` and the
 :ref:`shape-fill-tool`, also have a mode where they can :ref:`fill an area with
@@ -335,7 +335,7 @@ your own project. A few things to keep in mind:
   the terrain overlay is displayed correctly, set up the *Orientation*,
   *Grid Width* and *Grid Height* in the tileset properties.
 
-- The tool will handle any number of terrains (up to 255) and each corner of a
+- The tool will handle any number of terrains (up to 254) and each corner of a
   tile can have a different type of terrain. Still, there are other ways of
   dealing with transitions that this tool can't handle. Also, it is not able
   to edit multiple layers at the same time. For a more flexible, but also more

diff --git a/docs/reference/tmx-map-format.rst b/docs/reference/tmx-map-format.rst
@@ -359,7 +359,7 @@ Defines a list of colors and any number of Wang tiles using these colors.
 
 Can contain at most one: :ref:`tmx-properties`
 
-Can contain up to 255: :ref:`tmx-wangcolor` (since Tiled 1.5)
+Can contain up to 254: :ref:`tmx-wangcolor` (255 since Tiled 1.5, 254 since Tiled 1.10.2)
 
 Can contain any number: :ref:`tmx-wangtile`
 
@@ -389,7 +389,7 @@ associating it with a certain Wang ID.
 
 -  **tileid:** The tile ID.
 -  **wangid:** The Wang ID, since Tiled 1.5 given by a comma-separated list of
-   indexes (0-255) referring to the Wang colors in the Wang set in the order:
+   indexes (0-254) referring to the Wang colors in the Wang set in the order:
    top, top-right, right, bottom-right, bottom, bottom-left, left, top-left.
    Index 0 means *unset* and index 1 refers to the first Wang color. Before
    Tiled 1.5, the Wang ID was saved as a 32-bit unsigned integer stored in the

diff --git a/src/libtiled/grid.h b/src/libtiled/grid.h
@@ -60,6 +60,9 @@ class Grid
             mValues(CHUNK_SIZE * CHUNK_SIZE)
         {}
 
+        T &get(int x, int y) { return mValues[x + y * CHUNK_SIZE]; }
+        T &get(QPoint point) { return get(point.x(), point.y()); }
+
         const T &get(int x, int y) const { return mValues.at(x + y * CHUNK_SIZE); }
         const T &get(QPoint point) const { return get(point.x(), point.y()); }
 
@@ -102,6 +105,20 @@ class Grid
         return get(point.x(), point.y());
     }
 
+    T &add(int x, int y)
+    {
+        Chunk *chunk = findChunk(x, y);
+        if (!chunk)
+            chunk = &this->chunk(x, y);
+
+        return chunk->get(x & CHUNK_MASK, y & CHUNK_MASK);
+    }
+
+    T &add(QPoint point)
+    {
+        return add(point.x(), point.y());
+    }
+
     /**
      * Sets the value at the given coordinates.
      */

diff --git a/src/libtiled/wangset.cpp b/src/libtiled/wangset.cpp
@@ -718,60 +718,6 @@ QList<WangTile> WangSet::sortedWangTiles() const
     return wangTiles;
 }
 
-/**
- * Returns a WangId matching that of the provided \a surroundingWangIds.
- *
- * This is based off a provided array, { 0, 1, 2, 3, 4, 5, 6, 7 },
- * which corresponds to:
- *
- *      7|0|1
- *      6|X|2
- *      5|4|3
- */
-WangId WangSet::wangIdFromSurrounding(const WangId surroundingWangIds[])
-{
-    quint64 id = 0;
-
-    // Edges
-    for (int i = 0; i < WangId::NumEdges; ++i)
-        id |= quint64(surroundingWangIds[i*2].edgeColor((2 + i) % WangId::NumEdges)) << (i * WangId::BITS_PER_INDEX * 2);
-
-    // Corners
-    for (int i = 0; i < WangId::NumCorners; ++i) {
-        int color = surroundingWangIds[i*2 + 1].cornerColor((2 + i) % WangId::NumCorners);
-
-        if (!color)
-            color = surroundingWangIds[i*2].cornerColor((1 + i) % WangId::NumCorners);
-
-        if (!color)
-            color = surroundingWangIds[(i*2 + 2) % WangId::NumIndexes].cornerColor((3 + i) % WangId::NumCorners);
-
-        id |= quint64(color) << (WangId::BITS_PER_INDEX + i * WangId::BITS_PER_INDEX * 2);
-    }
-
-    return id;
-}
-
-/**
- * Returns a wangId matching that of the provided surrounding tiles.
- *
- * This is based off a provided array, { 0, 1, 2, 3, 4, 5, 6, 7 },
- * which corresponds to:
- *
- *      7|0|1
- *      6|X|2
- *      5|4|3
- */
-WangId WangSet::wangIdFromSurrounding(const Cell surroundingCells[]) const
-{
-    WangId wangIds[WangId::NumIndexes];
-
-    for (int i = 0; i < WangId::NumIndexes; ++i)
-        wangIds[i] = wangIdOfCell(surroundingCells[i]);
-
-    return wangIdFromSurrounding(wangIds);
-}
-
 /**
  * Returns the WangId of a given \a tile.
  *

diff --git a/src/libtiled/wangset.h b/src/libtiled/wangset.h
@@ -47,7 +47,7 @@ class TILEDSHARED_EXPORT WangId
     constexpr static unsigned BITS_PER_INDEX = 8;
     constexpr static quint64 INDEX_MASK = 0xFF;
     constexpr static quint64 FULL_MASK = Q_UINT64_C(0xFFFFFFFFFFFFFFFF);
-    constexpr static int MAX_COLOR_COUNT = (1 << BITS_PER_INDEX) - 1;
+    constexpr static int MAX_COLOR_COUNT = (1 << BITS_PER_INDEX) - 2;
 
     enum Index {
         Top         = 0,
@@ -278,9 +278,6 @@ class TILEDSHARED_EXPORT WangSet : public Object
 
     QList<WangTile> sortedWangTiles() const;
 
-    static WangId wangIdFromSurrounding(const WangId surroundingWangIds[]);
-    WangId wangIdFromSurrounding(const Cell surroundingCells[]) const;
-
     WangId wangIdOfTile(const Tile *tile) const;
     WangId wangIdOfCell(const Cell &cell) const;
 

diff --git a/src/tiled/abstracttilefilltool.cpp b/src/tiled/abstracttilefilltool.cpp
@@ -286,9 +286,9 @@ void AbstractTileFillTool::wangFill(TileLayer &tileLayerToFill,
     if (!mWangSet)
         return;
 
-    WangFiller wangFiller(*mWangSet, mapDocument()->renderer());
+    WangFiller wangFiller(*mWangSet, backgroundTileLayer, mapDocument()->renderer());
     wangFiller.setRegion(region);
-    wangFiller.apply(tileLayerToFill, backgroundTileLayer);
+    wangFiller.apply(tileLayerToFill);
 }
 
 void AbstractTileFillTool::fillWithStamp(Map &map,

diff --git a/src/tiled/randompicker.h b/src/tiled/randompicker.h
@@ -57,6 +57,11 @@ class RandomPicker
         return mThresholds.isEmpty();
     }
 
+    qsizetype size() const
+    {
+        return mThresholds.size();
+    }
+
     const T &pick() const
     {
         Q_ASSERT(!isEmpty());

diff --git a/src/tiled/stampbrush.cpp b/src/tiled/stampbrush.cpp
@@ -461,9 +461,9 @@ void StampBrush::drawPreviewLayer(const QVector<QPoint> &points)
             new TileLayer(QString(), bounds.topLeft(), bounds.size())
         };
 
-        WangFiller wangFiller(*mWangSet, mapDocument()->renderer());
+        WangFiller wangFiller(*mWangSet, *tileLayer, mapDocument()->renderer());
         wangFiller.setRegion(paintedRegion);
-        wangFiller.apply(*previewLayer, *tileLayer);
+        wangFiller.apply(*previewLayer);
 
         preview->addLayer(std::move(previewLayer));
         preview->addTileset(mWangSet->tileset()->sharedFromThis());

diff --git a/src/tiled/tilelayerwangedit.cpp b/src/tiled/tilelayerwangedit.cpp
@@ -38,6 +38,7 @@ TileLayerWangEdit::TileLayerWangEdit(EditableTileLayer *tileLayer, EditableWangS
     , mMap(tileLayer->map()->map()->parameters())
     , mRenderer(MapRenderer::create(&mMap))
     , mWangFiller(std::make_unique<WangFiller>(*wangSet->wangSet(),
+                                               *mTargetLayer->tileLayer(),
                                                mRenderer.get()))
 {
     mTargetLayer->mActiveWangEdits.append(this);
@@ -90,7 +91,7 @@ void TileLayerWangEdit::setEdge(QPoint pos, WangIndex::Value edge, int color)
 EditableTileLayer *TileLayerWangEdit::generate()
 {
     auto changes = std::make_unique<TileLayer>();
-    mWangFiller->apply(*changes, *mTargetLayer->tileLayer());
+    mWangFiller->apply(*changes);
     return new EditableTileLayer(std::move(changes));
 }
 
@@ -102,7 +103,7 @@ void TileLayerWangEdit::apply()
 
     // Apply terrain changes
     TileLayer changes;
-    mWangFiller->apply(changes, *mTargetLayer->tileLayer());
+    mWangFiller->apply(changes);
     mTargetLayer->applyChangesFrom(&changes, mergeable);
 }
 

diff --git a/src/tiled/wangbrush.cpp b/src/tiled/wangbrush.cpp
@@ -517,7 +517,7 @@ void WangBrush::updateBrush()
     const TileLayer *currentLayer = currentTileLayer();
     Q_ASSERT(currentLayer);
 
-    WangFiller wangFiller { *mWangSet, mapDocument()->renderer() };
+    WangFiller wangFiller { *mWangSet, *currentLayer, mapDocument()->renderer() };
 
     QVector<QPoint> points;
     bool ignoreFirst = false;
@@ -599,7 +599,7 @@ void WangBrush::updateBrush()
     SharedTileLayer stamp = SharedTileLayer::create(QString(), 0, 0, 0, 0);
 
     wangFiller.setCorrectionsEnabled(true);
-    wangFiller.apply(*stamp, *currentLayer);
+    wangFiller.apply(*stamp);
 
     static_cast<WangBrushItem*>(brushItem())->setInvalidTiles();
 
@@ -617,9 +617,6 @@ void WangBrush::updateBrush()
 void WangBrush::updateBrushAt(WangFiller &filler, QPoint pos)
 {
     auto hexagonalRenderer = dynamic_cast<HexagonalRenderer*>(mapDocument()->renderer());
-    auto &fill = filler.region();
-    Grid<WangFiller::CellInfo> &grid = fill.grid;
-    QRegion &region = fill.region;
 
     // When drawing lines in PaintEdgeAndCorner mode we force "tile mode"
     // because we currently can't draw thinner lines properly in that mode.
@@ -648,7 +645,7 @@ void WangBrush::updateBrushAt(WangFiller &filler, QPoint pos)
                 adjacentPositions[i] = pos + aroundTilePoints[i];
         }
 
-        WangFiller::CellInfo center = grid.get(pos);
+        WangFiller::CellInfo &center = filler.changePosition(pos);
 
         switch (mBrushMode) {
         case PaintCorner:
@@ -673,16 +670,13 @@ void WangBrush::updateBrushAt(WangFiller &filler, QPoint pos)
             break;
         }
 
-        region += QRect(pos, QSize(1, 1));
-        grid.set(pos, center);
-
         for (int i = 0; i < WangId::NumIndexes; ++i) {
             const bool isCorner = WangId::isCorner(i);
             if (mBrushMode == PaintEdge && isCorner)
                 continue;
 
             QPoint p = adjacentPositions[i];
-            WangFiller::CellInfo adjacent = grid.get(p);
+            WangFiller::CellInfo &adjacent = filler.changePosition(p);
 
             // Mark the opposite side or corner of the adjacent tile
             if (isCorner || (mBrushMode == PaintEdge || mBrushMode == PaintEdgeAndCorner)) {
@@ -697,9 +691,6 @@ void WangBrush::updateBrushAt(WangFiller &filler, QPoint pos)
                 adjacent.mask.setIndexColor((i + 3) % WangId::NumIndexes, WangId::INDEX_MASK);
                 adjacent.mask.setIndexColor((i + 5) % WangId::NumIndexes, WangId::INDEX_MASK);
             }
-
-            region += QRect(p, QSize(1, 1));
-            grid.set(p, adjacent);
         }
     } else {
         if (mWangIndex == WangId::NumIndexes)