Maproulette: Rebuild all; Enable script to handle server failures

Ping maproulette/maproulette3#2569
FixMyBerlin · Feb 26, 2025 · 1e90bde · 1e90bde
1 parent 2f34bbd
commit 1e90bde
Show file tree

Hide file tree

Showing 20 changed files with 157 additions and 43 deletions.
diff --git a/app/package.json b/app/package.json
@@ -49,7 +49,7 @@
     "MapboxTilesets:update": "bun ./scripts/MapboxTilesets/process.ts",
     "MaprouletteCreate": "bun run --env-file=scripts/MaprouletteCreate/.env scripts/MaprouletteCreate/process.ts",
     "MaprouletteRebuild": "bun run --env-file=scripts/MaprouletteCreate/.env scripts/MaprouletteRebuild/process.ts",
-    "Maproulette:help": "echo -e '\\033[0;32m Help: Use `npm run maproulette:process -- --filter FOOBAR` \\033[0m' && echo -e '\\033[0;32m Rebuild: Use `npm run maproulette:rebuild -- --filter FOOBAR` \\033[0m'",
+    "Maproulette:help": "echo -e '\\033[0;32m Help: Use `npm run MaprouletteCreate -- --filter FOOBAR` \\033[0m' && echo -e '\\033[0;32m Rebuild: Use `npm run MaprouletteRebuild -- --filter FOOBAR` \\033[0m'",
     "updatePackages:major": "npx taze major --includeLocked --write --exclude eslint,minimatch && npm run updatePackages:minor && npm install",
     "updatePackages:minor": "npx taze minor --includeLocked --write && npm install",
     "release": "gh pr create --base main --head develop --title \"Release $(date '+%Y-%m-%d')\" --body \"\"",

diff --git a/app/scripts/MaprouletteCreate/default.const.ts b/app/scripts/MaprouletteCreate/default.const.ts
@@ -5,7 +5,7 @@ export const defaultChallenge = {
   defaultBasemap: -1,
   defaultBasemapId: '',
   instruction:
-    ' \n## Kontext {{osmIdentifier}} \n \n{{task_markdown}}\n(Letzte Aktualisierung der Aufgabe: {{task_updated_at}})\n(Letzte Aktualisierung der Daten: {{data_updated_at}})\n {{blank}}          {{blank}}                  {{blank}}',
+    ' \n## Kontext {{osmIdentifier}} \n \n{{task_markdown}}\n\n(Letzte Aktualisierung der Aufgabe: {{task_updated_at}})\n(Letzte Aktualisierung der Daten: {{data_updated_at}})\n {{blank}}          {{blank}}                  {{blank}}',
   difficulty: 2,
   defaultPriority: 0,
   // Format: {"condition":"AND","rules":[{"value":"priority.prio1","type":"string","operator":"equal"}]}

diff --git a/app/scripts/MaprouletteRebuild/process.ts b/app/scripts/MaprouletteRebuild/process.ts
@@ -1,7 +1,10 @@
-import { campaigns } from '@/src/data/radinfra-de/campaigns'
+import { campaignsIncludingTest } from '@/src/data/radinfra-de/campaigns'
 import { CampaignMaprouletteSchema } from '@/src/data/radinfra-de/schema/campaignsSchema'
+import chalk from 'chalk'
 import { parseArgs } from 'util'
 import { maprouletteChallengeUrl } from '../MaprouletteCreate/utils/maprouletteChallengeUrl'
+import { ChallengeStatus } from './utils/challengeStatus'
+const { blue, gray, green, red, white, yellow } = chalk
 
 // https://bun.sh/guides/process/argv
 const { values } = parseArgs({
@@ -13,42 +16,126 @@ const { values } = parseArgs({
   allowPositionals: true,
 })
 
+// We check the status of the challenge in order to learn if the rebuild actually worked
+// and in order to slow down the script so we don't overwhelm the MR server.
+// See https://github.com/maproulette/maproulette3/issues/2569
+const RETRY_TIME_MS = 10 * 1000
+const MAX_RETRIES = 50
+const checkChallengeStatus = async (campaignId: number, retries = 0) => {
+  const tab = '\t\t\t'
+
+  const url = `https://maproulette.org/api/v2/challenge/${campaignId}`
+  try {
+    const response = await fetch(url)
+    if (!response.ok) {
+      console.log(
+        tab,
+        red('❌'),
+        'Failed to fetch challenge status',
+        response.status,
+        response.statusText,
+      )
+      return 'max_retries_exceeded'
+    }
+
+    const challenge = await response.json()
+    const challengeStatus = challenge.status
+
+    if (retries >= MAX_RETRIES) {
+      console.log(tab, red(`Challenge build failed after ${MAX_RETRIES} retries.`))
+      return 'failed'
+    }
+
+    const msgWaiting = `Waiting for ${Math.round(RETRY_TIME_MS / 1000)} seconds…`
+    const msgRetry = `${retries} / ${MAX_RETRIES}`
+    switch (challengeStatus) {
+      case ChallengeStatus.building:
+        console.log(tab, blue('Challenge is rebuilding tasks.'), msgWaiting, msgRetry)
+        await Bun.sleep(RETRY_TIME_MS)
+        return checkChallengeStatus(campaignId, retries + 1)
+      case ChallengeStatus.failed:
+        console.log(tab, red('Challenge build failed.'))
+        return 'failed'
+      case ChallengeStatus.ready:
+        console.log(tab, green('Challenge is ready.'))
+        return 'ready'
+      case ChallengeStatus.partiallyLoaded:
+        console.log(tab, blue('Challenge is partially loaded.'), msgWaiting, msgRetry)
+        await Bun.sleep(RETRY_TIME_MS)
+        return checkChallengeStatus(campaignId, retries + 1)
+      case ChallengeStatus.finished:
+        console.log(tab, green('Challenge is finished.'))
+        console.log(
+          '\t\t',
+          yellow('NOW: Update this challenge to be completed'),
+          maprouletteChallengeUrl(campaignId),
+        )
+        return 'finished'
+      case ChallengeStatus.deletingTasks:
+        console.log(tab, blue('Challenge is deleting tasks.'), msgWaiting, msgRetry)
+        await Bun.sleep(RETRY_TIME_MS)
+        return checkChallengeStatus(campaignId, retries + 1)
+      case ChallengeStatus.none:
+      case ChallengeStatus.empty:
+      default:
+        console.log(tab, red('ℹ️'), 'Challenge has no status or unknown status.')
+        return 'failed'
+    }
+  } catch (error) {
+    console.log(tab, red('Error fetching challenge status:'), error)
+    return 'failed'
+  }
+}
+
 async function main(filter: string | undefined) {
-  for await (const campaign of campaigns) {
+  for await (const campaign of campaignsIncludingTest) {
     // SKIP WHEN MR OFF
     if (campaign.maprouletteChallenge.enabled === false) {
-      console.log('\t', '\x1b[37m↷ SKIP\x1b[0m', campaign.id)
+      console.log('\t', white('↷ SKIP'), campaign.id)
       continue
     }
 
     const saveParsed = CampaignMaprouletteSchema.parse(campaign) // Second time gets rid of the enabled=false
+    const campaignId = saveParsed.maprouletteChallenge.id
+    const challengeUrl = maprouletteChallengeUrl(campaignId)
+    if (!campaignId) {
+      console.log('\t', yellow('↷ SKIP'), 'No campaignId', campaignId)
+      continue
+    }
 
     // SKIP BY FILTER PARAM
     const skip = filter ? !campaign.id.includes(filter) : false
-    const logPrefix = skip ? '\x1b[33m↷ SKIP\x1b[0m' : '\x1b[32m✎ PROCESS\x1b[0m'
-    console.log('\t', logPrefix, campaign.id, skip ? saveParsed.maprouletteChallenge.rebuildAt : '')
+    const logPrefix = skip ? yellow('↷ SKIP') : green('✎ PROCESS')
+    console.log('\t', logPrefix, campaign.id, gray(challengeUrl))
     if (skip) continue
 
     // ACTION
-    const campaignId = saveParsed.maprouletteChallenge.id
-    const url = `https://maproulette.org/api/v2/challenge/${campaignId}/rebuild?removeUnmatched=true&skipSnapshot=false`
-    const response = await fetch(url, {
+    const apiUrl = `https://maproulette.org/api/v2/challenge/${campaignId}/rebuild?removeUnmatched=true&skipSnapshot=true`
+    const response = await fetch(apiUrl, {
       method: 'PUT',
-      headers: {
-        accept: '*/*',
-        apiKey: process.env.MAPROULETTE_KEY!,
-      },
+      headers: { apiKey: process.env.MAPROULETTE_KEY!, accept: '*/*' },
     })
 
     if (!response.ok) {
-      const error = `Failed to trigger rebuild for challenge: ${response.statusText}`
-      console.error('\t\t', error, response, response)
-      continue
+      // https://github.com/maproulette/maproulette3/issues/2569
+      if (response.status === 502 || response.status === 400) {
+        const msgError = yellow('Server responded with "expected" error')
+        console.log('\t\t', msgError, response.status, response.statusText)
+
+        const status = await checkChallengeStatus(campaignId)
+        if (status === 'failed') {
+          console.log('\t\t', red('Rebuild failed for campaign'), campaign.id)
+          continue
+        }
+      } else {
+        const msgError = red('Failed to trigger rebuild for challenge')
+        console.error('\t\t', msgError, response.statusText, response)
+        continue
+      }
     }
 
-    // TODO: Readd the part where we write back teh rebuildAt to the local data automatically
-    console.log('\t\t', 'TRIGGERED REBUILD for campaign', maprouletteChallengeUrl(campaignId))
-    console.log('\t\t', `NOW UPDATE campaign ${campaign.id} WITH`, {
+    // TODO: Find a way to store this automatically
+    console.log('\t\t', `REBUILD FINISHED. NOW UPDATE campaign ${campaign.id} WITH`, {
       rebuildAt: new Date().toISOString(),
     })
   }
@@ -59,6 +146,6 @@ async function main(filter: string | undefined) {
 
 console.log(
   'STARTING MaprouletteRebuild',
-  values.filter ? `–\x1b[33m using filter \"${values.filter}\"\x1b[0m` : '',
+  values.filter ? `– ${yellow(`using filter "${values.filter}"`)}` : '',
 )
 main(values.filter)
diff --git a/app/scripts/MaprouletteRebuild/utils/challengeStatus.ts b/app/scripts/MaprouletteRebuild/utils/challengeStatus.ts
@@ -0,0 +1,28 @@
+// SOURCE
+// https://github.com/maproulette/maproulette3/blob/main/src/services/Challenge/ChallengeStatus/ChallengeStatus.js#L5-L15
+
+/**
+ * Constants defining the various statuses a Challenge may be
+ * in. These statuses are defined on the server
+ */
+const CHALLENGE_STATUS_NONE = 0 // called NA on the server
+const CHALLENGE_STATUS_BUILDING = 1
+const CHALLENGE_STATUS_FAILED = 2
+const CHALLENGE_STATUS_READY = 3
+const CHALLENGE_STATUS_PARTIALLY_LOADED = 4
+const CHALLENGE_STATUS_FINISHED = 5
+const CHALLENGE_STATUS_DELETING_TASKS = 6
+
+// We can ask the server for challenges that have no status by using -1
+const CHALLENGE_STATUS_EMPTY = -1
+
+export const ChallengeStatus = Object.freeze({
+  none: CHALLENGE_STATUS_NONE,
+  building: CHALLENGE_STATUS_BUILDING,
+  failed: CHALLENGE_STATUS_FAILED,
+  ready: CHALLENGE_STATUS_READY,
+  partiallyLoaded: CHALLENGE_STATUS_PARTIALLY_LOADED,
+  finished: CHALLENGE_STATUS_FINISHED,
+  deletingTasks: CHALLENGE_STATUS_DELETING_TASKS,
+  empty: CHALLENGE_STATUS_EMPTY,
+})
diff --git a/app/src/data/radinfra-de/campaigns.ts b/app/src/data/radinfra-de/campaigns.ts
@@ -14,6 +14,7 @@ import { missing_width } from './campaigns/missing_width'
 import { mixed_cycleway_both } from './campaigns/mixed_cycleway_both'
 import { needs_clarification } from './campaigns/needs_clarification'
 import { needs_clarification_track } from './campaigns/needs_clarification_track'
+import { test_maproulette_updates } from './campaigns/test_maproulette_updates'
 import { unexpected_bicycle_access_on_footway } from './campaigns/unexpected_bicycle_access_on_footway'
 import { CampaignSchema } from './schema/campaignsSchema'
 
@@ -37,7 +38,7 @@ const rawCampaigns = [
   deprecated_cycleway_shared,
 ]
 
-const collectCampaigns = () => {
+const collectCampaigns = (rawCampaigns) => {
   return rawCampaigns
     .map((campaign) => {
       const parsed = CampaignSchema.safeParse(campaign)
@@ -50,4 +51,5 @@ const collectCampaigns = () => {
     .filter(Boolean)
 }
 
-export const campaigns = collectCampaigns()
+export const campaigns = collectCampaigns(rawCampaigns)
+export const campaignsIncludingTest = collectCampaigns([...rawCampaigns, test_maproulette_updates])
diff --git a/app/src/data/radinfra-de/campaigns/adjoining_or_isolated.ts b/app/src/data/radinfra-de/campaigns/adjoining_or_isolated.ts
@@ -16,7 +16,7 @@ export const adjoining_or_isolated = {
       'Angabe zur Führung der Fahrradinfrastruktur (straßenbegleitend bzw. selbstständig geführt) ergänzt.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-18T10:46',
+    rebuildAt: '2025-02-26T13:47:15.793Z',
   },
   taskTemplate: `
 Für diese Infrastruktur (%%CATEGORY%%) fehlt uns eine Angabe, ob sie straßenbegleitend ist (oder nicht).

diff --git a/app/src/data/radinfra-de/campaigns/advisory_or_exclusive.ts b/app/src/data/radinfra-de/campaigns/advisory_or_exclusive.ts
@@ -15,7 +15,7 @@ export const advisory_or_exclusive = {
     checkinComment: 'Angabe zur Führungsform ergänzt (Schutzstreifen oder Angebotsstreifen).',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-14T12:51',
+    rebuildAt: '2025-02-26T13:50:18.013Z',
   },
   taskTemplate: `
 Für diese Infrastruktur fehlen uns Angaben, um sie als Schutzstreifen oder Radfahrstreifen einzutragen.

diff --git a/app/src/data/radinfra-de/campaigns/currentness_too_old.ts b/app/src/data/radinfra-de/campaigns/currentness_too_old.ts
@@ -14,7 +14,7 @@ export const currentness_too_old = {
     checkinComment: 'Radinfrastruktur überprüft.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-14T13:03',
+    rebuildAt: '2025-02-26T13:50:42.219Z',
   },
   taskTemplate: `
 Dieser Weg ist seit vielen Jahren nicht mehr überprüft worden.

diff --git a/app/src/data/radinfra-de/campaigns/deprecated_cycleway_shared.ts b/app/src/data/radinfra-de/campaigns/deprecated_cycleway_shared.ts
@@ -14,7 +14,7 @@ export const deprecated_cycleway_shared = {
     checkinComment: 'Deprecated cycleway=shared.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-18T10:54',
+    rebuildAt: '2025-02-26T13:55:05.955Z',
   },
   taskTemplate: `
 Dieser Weg hat den veralteten Tag \`cycleway=shared\`. Diese wollen wir aktualisieren.

diff --git a/app/src/data/radinfra-de/campaigns/malformed_traffic_sign.ts b/app/src/data/radinfra-de/campaigns/malformed_traffic_sign.ts
@@ -14,7 +14,7 @@ export const malformed_traffic_sign = {
     checkinComment: '`traffic_sign:*` korrigiert.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-18T10:54',
+    rebuildAt: '2025-02-26T13:53:42.353Z',
   },
   taskTemplate: `
 Das Verkehrszeichen-Tag an diese Weg enthält einen Fehler.

diff --git a/app/src/data/radinfra-de/campaigns/missing_access_tag_240.ts b/app/src/data/radinfra-de/campaigns/missing_access_tag_240.ts
@@ -15,7 +15,7 @@ export const missing_access_tag_240 = {
     checkinComment: 'Access-Tags ergänzt.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-18T10:54',
+    rebuildAt: '2025-02-26T13:50:15.195Z',
   },
   taskTemplate: `
 Für diesen Weg wurde das Verkehrszeichen \`240\` oder \`241\` angegeben aber ein entsprechendes Zugangs-Taggging fehlt.

diff --git a/app/src/data/radinfra-de/campaigns/missing_access_tag_bicycle_road.ts b/app/src/data/radinfra-de/campaigns/missing_access_tag_bicycle_road.ts
@@ -14,7 +14,7 @@ export const missing_access_tag_bicycle_road = {
     checkinComment: 'bicycle=designated an Fahrradstraßen ergänzt.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-14T13:04',
+    rebuildAt: '2025-02-26T13:50:15.376Z',
   },
   taskTemplate: `
 Dieser Weg ist als Fahrradstraße getaggt **ohne das zugehörige \`bicycle=designated\`.** Ohne diesen Zusatz können Router nicht erkennen, dass die Straße für den Radverkehr freigegeben ist.

diff --git a/app/src/data/radinfra-de/campaigns/missing_segregated.ts b/app/src/data/radinfra-de/campaigns/missing_segregated.ts
@@ -15,7 +15,7 @@ export const missing_segregated = {
     checkinComment: 'Angabe zu `segregated` ergänzt.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-14T13:04',
+    rebuildAt: '2025-02-26T13:50:15.376Z',
   },
   taskTemplate: `
 Dieser Weg wird vom Fußverkehr und Radverkehr genutzt (laut Verkehrszeichen oder Zugangs-Tagging).

diff --git a/app/src/data/radinfra-de/campaigns/missing_traffic_sign_244.ts b/app/src/data/radinfra-de/campaigns/missing_traffic_sign_244.ts
@@ -14,7 +14,7 @@ export const missing_traffic_sign_244 = {
     checkinComment: 'Verkehrszeichen an Fahrradstraße ergänzt.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-18T10:54',
+    rebuildAt: '2025-02-26T13:53:46.823Z',
   },
   taskTemplate: `
 Dieser Weg ist als Fahrradstraße getaggt, **jedoch fehlt der zugehörige Verkehrszeichen-Tag.**

diff --git a/app/src/data/radinfra-de/campaigns/missing_traffic_sign_vehicle_destination.ts b/app/src/data/radinfra-de/campaigns/missing_traffic_sign_vehicle_destination.ts
@@ -15,7 +15,7 @@ export const missing_traffic_sign_vehicle_destination = {
     checkinComment: 'Verkehrszeichen und access-Tagging vereinheitlicht.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-14T12:57',
+    rebuildAt: '2025-02-26T13:53:50.238Z',
   },
   taskTemplate: `
 Dieser Weg ist als Fahrradstraße mit Freigabe für Kfz getaggt, **jedoch fehlt der zugehörige Verkehrszeichen-Tag (Zusatzzeichen).**

diff --git a/app/src/data/radinfra-de/campaigns/mixed_cycleway_both.ts b/app/src/data/radinfra-de/campaigns/mixed_cycleway_both.ts
@@ -15,7 +15,7 @@ export const mixed_cycleway_both = {
     checkinComment: 'Überschneidende Tags präzisiert.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-18T10:54',
+    rebuildAt: '2025-02-26T13:50:18.013Z',
   },
   taskTemplate: `
 Dieser Weg hat Tags die sich überschneiden:

diff --git a/app/src/data/radinfra-de/campaigns/needs_clarification.ts b/app/src/data/radinfra-de/campaigns/needs_clarification.ts
@@ -15,7 +15,7 @@ export const needs_clarification = {
     checkinComment: 'Angabe zur Führungsform ergänzt.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-14T12:58',
+    rebuildAt: '2025-02-26T13:55:06.091Z',
   },
   taskTemplate: `
 Diese Radinfrastruktur konnte nicht richtig kategorisiert werden.

diff --git a/app/src/data/radinfra-de/campaigns/test_maproulette_updates.ts b/app/src/data/radinfra-de/campaigns/test_maproulette_updates.ts
@@ -15,7 +15,7 @@ export const test_maproulette_updates = {
     checkinComment: '#radinfra_de #maproulette',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-18T10:47',
+    rebuildAt: '2025-02-26T13:55:06.091Z',
   },
   taskTemplate: ``,
 }
diff --git a/app/src/data/radinfra-de/campaigns/unexpected_bicycle_access_on_footway.ts b/app/src/data/radinfra-de/campaigns/unexpected_bicycle_access_on_footway.ts
@@ -15,7 +15,7 @@ export const unexpected_bicycle_access_on_footway = {
     checkinComment: 'Straßenklasse Fußweg in Kombination mit Radinfrastruktur verbessert.',
     checkinSource: 'radinfra_de',
     resultsLimited: false,
-    rebuildAt: '2025-02-18T10:46',
+    rebuildAt: '2025-02-26T13:50:42.219Z',
   },
   taskTemplate: `
 Dieser Weg verwendet eine unerwarte Kombination an Tags: Er ist als Gehweg attributiert aber gleichzeitig als für Fahrrad vorgesehen Infrastruktur.

diff --git a/app/src/data/radinfra-de/schema/campaignsSchema.ts b/app/src/data/radinfra-de/schema/campaignsSchema.ts
@@ -27,6 +27,7 @@ const MaprouletteEnabled = z.object({
 const MaprouletteDisabled = z.object({ enabled: z.literal(false) })
 
 const CampaignBaseSchema = z.object({
+  id: z.string(),
   title: z.string(),
   pubDate: InputDateTimeSchema,
   visibility: z.enum(visibilityOptions),
@@ -37,10 +38,8 @@ const CampaignBaseSchema = z.object({
   task: z.string(),
   taskTemplate: z.string(),
 })
-const CampaignAddedPropertiesSchmea = z.object({
-  id: z.string(),
-})
-export const CampaignSchema = CampaignBaseSchema.merge(CampaignAddedPropertiesSchmea).merge(
+
+export const CampaignSchema = CampaignBaseSchema.merge(
   z.object({
     maprouletteChallenge: z.discriminatedUnion('enabled', [
       MaprouletteEnabled,
@@ -50,8 +49,6 @@ export const CampaignSchema = CampaignBaseSchema.merge(CampaignAddedPropertiesSc
 )
 
 export const CampaignMaprouletteSchema = CampaignBaseSchema.merge(
-  CampaignAddedPropertiesSchmea,
-).merge(
   z.object({
     maprouletteChallenge: MaprouletteEnabled,
   }),