Merge pull request #58982 from nirvn/osm_bp_2

[backport queued ltr][processing] Insure that processing algorithms are not used to breach the OSMF tile usage policy on bulk download

python/PyQt6/core/auto_generated/qgsmaplayer.sip.in

@@ -215,6 +215,13 @@ Returns the layer's data provider, it may be ``None``.
 %End
 
 
+    QgsProviderMetadata *providerMetadata() const;
+%Docstring
+Returns the layer data provider's metadata, it may be ``None``.
+
+.. versionadded:: 3.40
+%End
+
     void setShortName( const QString &shortName );
 %Docstring
 Sets the short name of the layer

python/PyQt6/core/auto_generated/qgsmaplayerutils.sip.in

@@ -9,6 +9,7 @@
 
 
 
+
 class QgsMapLayerUtils
 {
 %Docstring(signature="appended")
@@ -78,6 +79,13 @@ Specifically this method:
 - Removes any characters which are not alphanumeric or '_'.
 
 .. versionadded:: 3.28
+%End
+
+    static bool isOpenStreetMapLayer( QgsMapLayer *layer );
+%Docstring
+Returns ``True`` if the layer is served by OpenStreetMap server.
+
+.. versionadded:: 3.40
 %End
 
 };

python/core/auto_generated/qgsmaplayer.sip.in

@@ -215,6 +215,13 @@ Returns the layer's data provider, it may be ``None``.
 %End
 
 
+    QgsProviderMetadata *providerMetadata() const;
+%Docstring
+Returns the layer data provider's metadata, it may be ``None``.
+
+.. versionadded:: 3.40
+%End
+
     void setShortName( const QString &shortName );
 %Docstring
 Sets the short name of the layer

python/core/auto_generated/qgsmaplayerutils.sip.in

@@ -9,6 +9,7 @@
 
 
 
+
 class QgsMapLayerUtils
 {
 %Docstring(signature="appended")
@@ -78,6 +79,13 @@ Specifically this method:
 - Removes any characters which are not alphanumeric or '_'.
 
 .. versionadded:: 3.28
+%End
+
+    static bool isOpenStreetMapLayer( QgsMapLayer *layer );
+%Docstring
+Returns ``True`` if the layer is served by OpenStreetMap server.
+
+.. versionadded:: 3.40
 %End
 
 };

src/analysis/processing/qgsalgorithmrasterize.cpp

@@ -24,6 +24,8 @@
 
 #include "qgsalgorithmrasterize.h"
 #include "qgsprocessingparameters.h"
+#include "qgsprovidermetadata.h"
+#include "qgsmaplayerutils.h"
 #include "qgsmapthemecollection.h"
 #include "qgsrasterfilewriter.h"
 #include "qgsmaprenderercustompainterjob.h"
@@ -150,6 +152,67 @@ QVariantMap QgsRasterizeAlgorithm::processAlgorithm( const QVariantMap &paramete
   int height { yTileCount * tileSize };
   int nBands { transparent ? 4 : 3 };
 
+  int64_t totalTiles = 0;
+  for ( auto &layer : std::as_const( mMapLayers ) )
+  {
+    if ( QgsMapLayerUtils::isOpenStreetMapLayer( layer.get() ) )
+    {
+      if ( QgsRasterLayer *rasterLayer = qobject_cast<QgsRasterLayer *>( ( layer.get() ) ) )
+      {
+        const QList< double > resolutions = rasterLayer->dataProvider()->nativeResolutions();
+        if ( resolutions.isEmpty() )
+        {
+          continue;
+        }
+
+        if ( totalTiles == 0 )
+        {
+          const QgsCoordinateTransform ct( context.project()->crs(), rasterLayer->crs(), context.transformContext() );
+          QgsRectangle extentLayer;
+          try
+          {
+            extentLayer = ct.transform( extent );
+          }
+          catch ( QgsCsException & )
+          {
+            totalTiles = -1;
+            continue;
+          }
+
+          const double mapUnitsPerPixelLayer = extentLayer.width() / width;
+          int i;
+          for ( i = 0; i < resolutions.size() && resolutions.at( i ) < mapUnitsPerPixelLayer; i++ )
+          {
+          }
+
+          if ( i == resolutions.size() ||
+               ( i > 0 && resolutions.at( i ) - mapUnitsPerPixelLayer > mapUnitsPerPixelLayer - resolutions.at( i - 1 ) ) )
+          {
+            i--;
+          }
+
+          const int nbTilesWidth = std::ceil( extentLayer.width() / resolutions.at( i ) / 256 );
+          const int nbTilesHeight = std::ceil( extentLayer.height() / resolutions.at( i ) / 256 );
+          totalTiles = static_cast<int64_t>( nbTilesWidth ) * nbTilesHeight;
+        }
+        feedback->pushInfo( QStringLiteral( "%1" ).arg( totalTiles ) );
+
+        if ( totalTiles > MAXIMUM_OPENSTREETMAP_TILES_FETCH )
+        {
+          // Prevent bulk downloading of tiles from openstreetmap.org as per OSMF tile usage policy
+          feedback->pushWarning( QObject::tr( "Layer %1 will be skipped as the algorithm leads to bulk downloading behavior which is prohibited by the %2OpenStreetMap Foundation tile usage policy%3" ).arg( rasterLayer->name(), QString(), QString() ) );
+
+          layer->deleteLater();
+          std::vector<std::unique_ptr<QgsMapLayer>>::iterator position = std::find( mMapLayers.begin(), mMapLayers.end(), layer );
+          if ( position != mMapLayers.end() )
+          {
+            mMapLayers.erase( position );
+          }
+        }
+      }
+    }
+  }
+
   const QString driverName { QgsRasterFileWriter::driverForExtension( QFileInfo( outputLayerFileName ).suffix() ) };
   if ( driverName.isEmpty() )
   {

src/analysis/processing/qgsalgorithmrasterize.h

@@ -55,6 +55,8 @@ class QgsRasterizeAlgorithm : public QgsProcessingAlgorithm
 
     bool prepareAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override;
 
+    void checkLayersUsagePolicy( QgsProcessingFeedback *feedback );
+
   private:
 
     QMap<QString, QString> mMapThemeStyleOverrides;

src/analysis/processing/qgsalgorithmxyztiles.cpp

@@ -22,6 +22,8 @@
 #include "qgslayertree.h"
 #include "qgslayertreelayer.h"
 #include "qgsexpressioncontextutils.h"
+#include "qgsmaplayerutils.h"
+#include "qgsprovidermetadata.h"
 
 ///@cond PRIVATE
 
@@ -185,6 +187,23 @@ bool QgsXyzTilesBaseAlgorithm::prepareAlgorithm( const QVariantMap &parameters,
   return true;
 }
 
+void QgsXyzTilesBaseAlgorithm::checkLayersUsagePolicy( QgsProcessingFeedback *feedback )
+{
+  if ( mTotalTiles > MAXIMUM_OPENSTREETMAP_TILES_FETCH )
+  {
+    for ( QgsMapLayer *layer : std::as_const( mLayers ) )
+    {
+      if ( QgsMapLayerUtils::isOpenStreetMapLayer( layer ) )
+      {
+        // Prevent bulk downloading of tiles from openstreetmap.org as per OSMF tile usage policy
+        feedback->pushWarning( QObject::tr( "Layer %1 will be skipped as the algorithm leads to bulk downloading behavior which is prohibited by the %2OpenStreetMap Foundation tile usage policy%3" ).arg( layer->name(), QString(), QString() ) );
+        mLayers.removeAll( layer );
+        delete layer;
+      }
+    }
+  }
+}
+
 void QgsXyzTilesBaseAlgorithm::startJobs()
 {
   while ( mRendererJobs.size() < mThreadsNumber && !mMetaTiles.empty() )
@@ -281,21 +300,25 @@ QVariantMap QgsXyzTilesDirectoryAlgorithm::processAlgorithm( const QVariantMap &
   mOutputDir = outputDir;
   mTms = tms;
 
+  mTotalTiles = 0;
   for ( int z = mMinZoom; z <= mMaxZoom; z++ )
   {
     if ( feedback->isCanceled() )
       break;
 
     mMetaTiles += getMetatiles( mWgs84Extent, z, mMetaTileSize );
+    feedback->pushWarning( QObject::tr( "%1 tiles will be created for zoom level %2" ).arg( mMetaTiles.size() - mTotalTiles ).arg( z ) );
+    mTotalTiles = mMetaTiles.size();
   }
+  feedback->pushWarning( QObject::tr( "A total of %1 tiles will be created" ).arg( mTotalTiles ) );
+
+  checkLayersUsagePolicy( feedback );
 
   for ( QgsMapLayer *layer : std::as_const( mLayers ) )
   {
     layer->moveToThread( QThread::currentThread() );
   }
 
-  mTotalTiles = mMetaTiles.size();
-
   QEventLoop loop;
   // cppcheck-suppress danglingLifetime
   mEventLoop = &loop;
@@ -403,7 +426,10 @@ void QgsXyzTilesDirectoryAlgorithm::processMetaTile( QgsMapRendererSequentialJob
       j->deleteLater();
     }
     mRendererJobs.clear();
-    mEventLoop->exit();
+    if ( mEventLoop )
+    {
+      mEventLoop->exit();
+    }
     return;
   }
 
@@ -413,7 +439,10 @@ void QgsXyzTilesDirectoryAlgorithm::processMetaTile( QgsMapRendererSequentialJob
   }
   else if ( mMetaTiles.size() == 0 && mRendererJobs.size() == 0 )
   {
-    mEventLoop->exit();
+    if ( mEventLoop )
+    {
+      mEventLoop->exit();
+    }
   }
 }
 
@@ -470,22 +499,34 @@ QVariantMap QgsXyzTilesMbtilesAlgorithm::processAlgorithm( const QVariantMap &pa
                       .arg( mWgs84Extent.xMaximum() ).arg( mWgs84Extent.yMaximum() );
   mMbtilesWriter->setMetadataValue( "bounds",  boundsStr );
 
+  mTotalTiles = 0;
   for ( int z = mMinZoom; z <= mMaxZoom; z++ )
   {
     if ( feedback->isCanceled() )
       break;
 
     mMetaTiles += getMetatiles( mWgs84Extent, z, mMetaTileSize );
+    feedback->pushInfo( QObject::tr( "%1 tiles will be created for zoom level %2" ).arg( mMetaTiles.size() - mTotalTiles ).arg( z ) );
+    mTotalTiles = mMetaTiles.size();
   }
+  feedback->pushInfo( QObject::tr( "A total of %1 tiles will be created" ).arg( mTotalTiles ) );
 
-  mTotalTiles = mMetaTiles.size();
+  checkLayersUsagePolicy( feedback );
+
+  for ( QgsMapLayer *layer : std::as_const( mLayers ) )
+  {
+    layer->moveToThread( QThread::currentThread() );
+  }
 
   QEventLoop loop;
   // cppcheck-suppress danglingLifetime
   mEventLoop = &loop;
   startJobs();
   loop.exec();
 
+  qDeleteAll( mLayers );
+  mLayers.clear();
+
   QVariantMap results;
   results.insert( QStringLiteral( "OUTPUT_FILE" ), outputFile );
   return results;
@@ -525,7 +566,10 @@ void QgsXyzTilesMbtilesAlgorithm::processMetaTile( QgsMapRendererSequentialJob *
       j->deleteLater();
     }
     mRendererJobs.clear();
-    mEventLoop->exit();
+    if ( mEventLoop )
+    {
+      mEventLoop->exit();
+    }
     return;
   }
 
@@ -535,7 +579,10 @@ void QgsXyzTilesMbtilesAlgorithm::processMetaTile( QgsMapRendererSequentialJob *
   }
   else if ( mMetaTiles.size() == 0 && mRendererJobs.size() == 0 )
   {
-    mEventLoop->exit();
+    if ( mEventLoop )
+    {
+      mEventLoop->exit();
+    }
   }
 }
 

src/analysis/processing/qgsalgorithmxyztiles.h

@@ -104,6 +104,8 @@ class QgsXyzTilesBaseAlgorithm : public QgsProcessingAlgorithm
 
     bool prepareAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback ) override;
 
+    void checkLayersUsagePolicy( QgsProcessingFeedback *feedback );
+
     void startJobs();
     virtual void processMetaTile( QgsMapRendererSequentialJob *job ) = 0;
 
@@ -129,7 +131,7 @@ class QgsXyzTilesBaseAlgorithm : public QgsProcessingAlgorithm
     long long mTotalTiles = 0;
     long long mProcessedTiles = 0;
     QgsCoordinateTransformContext mTransformContext;
-    QEventLoop *mEventLoop;
+    QPointer<QEventLoop> mEventLoop;
     QList< MetaTile > mMetaTiles;
     QMap< QgsMapRendererSequentialJob *, MetaTile > mRendererJobs;
 };

src/core/qgsmaplayer.cpp

@@ -33,6 +33,7 @@
 #include "qgsprojectfiletransform.h"
 #include "qgsproject.h"
 #include "qgsproviderregistry.h"
+#include "qgsprovidermetadata.h"
 #include "qgsrasterlayer.h"
 #include "qgsreadwritecontext.h"
 #include "qgsrectangle.h"
@@ -43,7 +44,6 @@
 #include "qgsmessagelog.h"
 #include "qgsmaplayertemporalproperties.h"
 #include "qgsmaplayerelevationproperties.h"
-#include "qgsprovidermetadata.h"
 #include "qgslayernotesutils.h"
 #include "qgsdatums.h"
 #include "qgsprojoperation.h"
@@ -236,6 +236,11 @@ const QgsDataProvider *QgsMapLayer::dataProvider() const
   return nullptr;
 }
 
+QgsProviderMetadata *QgsMapLayer::providerMetadata() const
+{
+  return QgsProviderRegistry::instance()->providerMetadata( providerType() );
+}
+
 QString QgsMapLayer::shortName() const
 {
   QGIS_PROTECT_QOBJECT_THREAD_ACCESS

src/core/qgsmaplayer.h

@@ -48,6 +48,7 @@ class QgsMapLayerLegend;
 class QgsMapLayerRenderer;
 class QgsMapLayerStyleManager;
 class QgsProject;
+class QgsProviderMetadata;
 class QgsStyleEntityVisitorInterface;
 class QgsMapLayerTemporalProperties;
 class QgsMapLayerElevationProperties;
@@ -285,6 +286,12 @@ class CORE_EXPORT QgsMapLayer : public QObject
      */
     virtual const QgsDataProvider *dataProvider() const SIP_SKIP;
 
+    /**
+     * Returns the layer data provider's metadata, it may be NULLPTR.
+     * \since QGIS 3.40
+     */
+    QgsProviderMetadata *providerMetadata() const;
+
     /**
      * Sets the short name of the layer
      *  used by QGIS Server to identify the layer.

src/core/qgsmaplayerutils.cpp

@@ -166,3 +166,20 @@ QString QgsMapLayerUtils::launderLayerName( const QString &name )
 
   return laundered;
 }
+
+bool QgsMapLayerUtils::isOpenStreetMapLayer( QgsMapLayer *layer )
+{
+  if ( layer->providerType() == QStringLiteral( "wms" ) )
+  {
+    if ( const QgsProviderMetadata *metadata = layer->providerMetadata() )
+    {
+      QVariantMap details = metadata->decodeUri( layer->source() );
+      QUrl url( details.value( QStringLiteral( "url" ) ).toString() );
+      if ( url.host().endsWith( QStringLiteral( ".openstreetmap.org" ) ) || url.host().endsWith( QStringLiteral( ".osm.org" ) ) )
+      {
+        return true;
+      }
+    }
+  }
+  return false;
+}