metadata: add mattb325's complete collection (#80)

* Add page 1

* Don't process packages already available

* Allow overriding local packages

* Add page 2

* Update 18253-blam-overpath-props-volume-1.yaml

* Update sc4d.js

* Create 29148-london-bridge-quarter-the-shard-and-place.yaml

* Add `xyloxadoria:6th-and-brushy`

* Esnure simfox day-and-nite is not included

* Add page 3

* Remove diggis ponds in favor of default channel

* Update sc4d.js

* Add page 4

* Create 34230-chinatown-road-avenue-and-pedestrian-lane-canopies.yaml

* Add page 5

* Update sc4d.js

* Add page 6

* Add page 7

* Update parse-dependencies.js

* Add page 8

* Add  page 9

* Add page 10

* Add page 11

* Add page 12

* Create 31893-car-ferry-terminal.yaml

* Update 31893-car-ferry-terminal.yaml

* Add page 13

* Add page 14

* Add page 15

* Create 27975-old-market-hall.yaml

* Fix missing subfolders

* Improve auto-detection of maxis and dark nite

* Fix MN/DN variants

* Fix MN/DN

* Fix model path too long

* Fix duplicate pwn-london

* Remove duplicates

We've removed a bunch of packages that are also included in memo33/sc4pac#47, so no need to put them on the ST channel too.

* Include filename when traversing

* Remove duplicates

We've removed a bunch of packages that are already available in memo33/sc4pac#46

* Remove duplicate

* Update sc4d.js

* Remove duplicates

Some packages were removed that are already present on the default channel.

* Fix exclusion pattern

* Delete 33006-diagonal-rail-glr-bus-and-subway-stations.yaml

* Add scripts for finding duplicates

actions/fetch/complete-metadata.js

@@ -47,8 +47,8 @@ export async function expandVariants(metadata) {
 
 // Regular expressions we use for detecting maxisnite or darknite *folders*.
 const regexes = {
-	maxisnite: /\b(maxisni(te|ght)|mn)\b/i,
-	darknite: /\b(darkni(te|ght)|dn)\b/i,
+	maxisnite: /\b(maxis ?ni(te|ght)|mn)\b/i,
+	darknite: /\b(dark ?ni(te|ght)|dn)\b/i,
 };
 
 // # findIncludedVariants(metadata)

actions/fetch/test/integration-test.js

@@ -488,6 +488,53 @@ describe('The fetch action', function() {
 
 	});
 
+	it('a package with MN and DN variants in the same upload (3)', async function() {
+
+		let upload = faker.upload({
+			author: 'mattb325',
+			title: 'Standard Federal Savings',
+			files: [
+				{
+					name: 'Standard Federal Savings.zip',
+					contents: [
+						'metadata.yaml',
+						'Standard Federal Savings DARK NITE/model.SC4Model',
+						'Standard Federal Savings DARK NITE/lot.SC4Lot',
+						'Standard Federal Savings DARK NITE/building.SC4Desc',
+						'Standard Federal Savings MAXIS NITE/model.SC4Model',
+						'Standard Federal Savings MAXIS NITE/lot.SC4Lot',
+						'Standard Federal Savings MAXIS NITE/building.SC4Desc',
+					],
+				},
+			],
+		});
+		const { run } = this.setup({ upload });
+
+		let { result } = await run({ id: upload.id });
+		expect(result.metadata[0].variants).to.eql([
+			{
+				variant: { nightmode: 'standard' },
+				assets: [
+					{
+						assetId: 'mattb325-standard-federal-savings',
+						exclude: ['/Standard Federal Savings DARK NITE/'],
+					},
+				],
+			},
+			{
+				variant: { nightmode: 'dark' },
+				dependencies: ['simfox:day-and-nite-mod'],
+				assets: [
+					{
+						assetId: 'mattb325-standard-federal-savings',
+						exclude: ['/Standard Federal Savings MAXIS NITE/'],
+					},
+				],
+			},
+		]);
+
+	});
+
 	it('a package with MN and DN variants in the same upload with nested folders', async function() {
 
 		let upload = faker.upload({

scripts/find-duplicates.js

@@ -0,0 +1,33 @@
+// # find-duplicates.js
+import { Glob } from 'glob';
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const glob = new Glob('**/*', {
+	cwd: process.env.SC4_PLUGINS,
+	nodir: true,
+	absolute: true,
+});
+let files = glob.walkSync();
+let map = {};
+for (let file of files) {
+	let hash = doHash(file);
+	map[hash] ??= [];
+	map[hash].push(path.relative(process.env.SC4_PLUGINS, file).replaceAll(path.sep, '/'));
+}
+
+function doHash(file) {
+	let contents = fs.readFileSync(file);
+	return crypto.createHash('sha256')
+		.update(contents)
+		.digest('hex')
+		.slice(0, 9);
+}
+
+let entries = Object.entries(map)
+	.filter(row => row[1].length > 1)
+	.filter(row => !row[1][0].includes('075-my-plugins'))
+	.filter(row => !row[1][0].includes('150-mods'))
+	.sort((a, b) => a[1].length - b[1].length);
+console.log(Object.fromEntries(entries));

scripts/list-matt-packages.js

@@ -0,0 +1,21 @@
+import path from 'node:path';
+import traverse from './traverse-yaml.js';
+
+let install = [];
+await traverse('yaml/mattb325/**/*.yaml', (json) => {
+	if (json.assetId) return;
+	install.push(`mattb325:${json.name}`);
+}, {
+	cwd: path.resolve(import.meta.dirname, '../../sc4pac/src'),
+});
+
+await traverse('yaml/mattb325/*.yaml', (json) => {
+	if (json.assetId) return;
+	let name = `${json.group}:${json.name}`;
+	install.push(name);
+});
+
+install.sort();
+for (let pkg of install) {
+	console.log(`    "${pkg}",`);
+}

scripts/manual-add.js

@@ -23,6 +23,22 @@ async function run(urls) {
 	// processed first.
 	urls = [urls].flat().sort();
 	let index = await buildIndex();
+
+	// Once we have the index, we'll still filter out the urls that are already 
+	// processed. They might either be present on the Simtropolis channel, or on 
+	// the default channel already.
+	urls = urls.filter(url => {
+		let id = urlToFileId(url);
+		let pkg = index.stex[id];
+		if (pkg && !pkg.local) {
+			console.log(styleText('yellow', `${url} is already present on one of the channels`));
+			return false;
+		} else {
+			return true;
+		}
+	});
+
+	// Cool, now perform the actual fetching.
 	let results = await fetchAll(urls);
 	for (let result of results) {
 		let [pkg] = result.metadata;
@@ -40,8 +56,7 @@ async function run(urls) {
 			let deps = parseDependencies(index, links);
 			let unmatched = deps.filter(dep => dep.startsWith('"['));
 			if (unmatched.length > 0) {
-				let id = `${pkg.group}:${pkg.name}`;
-				console.log(styleText('red', `${id} has unmatched dependencies that need to be fixed manually!`));
+				console.log(styleText('red', `${pkg.info.website} has unmatched dependencies that need to be fixed manually!`));
 				for (let dep of unmatched) {
 					console.log(`  ${styleText('cyan', dep)}`);
 				}
@@ -148,7 +163,9 @@ async function addFileToIndex(index, file) {
 			(index.stex[id] ??= new Map()).set(name, {
 				id: name,
 				subfolder: parsed.subfolder,
+				local: true,
 			});
+			index.stex[id].local = true;
 		}
 	}
 }

scripts/parse-dependencies.js

@@ -1,10 +1,15 @@
 // # parse-dependencies.js
+// Some urls that are often used as dependency, but which or not included on the 
+// channels, so we match them "as is".
 const urls = {
 	'http://kurier.simcityplaza.de/details.php?file=5': 'sfbt:essentials',
 	'http://kurier.simcityplaza.de/details.php?file=305': 'maxis:buildings-as-props',
 	'https://community.simtropolis.com/Documents/RR%20MEGA%20Prop%20Pack%20Vol.%202.3.3': 'rretail:mega-prop-pack-vol2',
 	'https://community.simtropolis.com/sc4-maxis-files/': 'maxis:buildings-as-props',
 	'https://www.simtropolis.com/stex/index.cfm?page=1&keyword=bsc%20peg&type=all': 'bsc:textures-vol01',
+	'https://community.simtropolis.com/files/file/30934-supershk-parking-textures-vol-1/': 'supershk:mega-parking-textures',
+	'https://community.simtropolis.com/files/file/11421-porkie-props-vol1-european-street-accessories/': 'porkissimo:jenx-porkie-expanded-porkie-props',
+	'https://community.simtropolis.com/files/file/27563-shk-parking-pack/': 'shk:parking-pack',
 };
 
 export default function parseDependencies(index, links) {

scripts/prune.js

@@ -22,7 +22,9 @@ for (let pkg of packages.explicit) {
 	if (standardDeps.includes(pkg)) continue;
 	spinner.text = `Tracking dependencies for ${pkg}`;
 	let result = await tracker.track(pkg);
-	index[pkg] = result.packages;
+	let set = new Set(result.packages);
+	set.delete('simfox:day-and-nite-mod');
+	result.packages = index[pkg] = [...set];
 	results.push(result);
 }
 spinner.succeed('Tracked all dependencies');
@@ -42,9 +44,7 @@ if (argv.force || argv.f) {
 
 		// If SimFox' day and nite mod was tracked as a dependency, this is because 
 		// a light cone was referenced with the mod active. We exclude this.
-		let set = new Set(deps);
-		set.delete('simfox:day-and-nite-mod');
-		pkg.dependencies = [...set].sort();
+		pkg.dependencies = deps;
 		return pkg;
 
 	});

scripts/sc4d.js

@@ -4,6 +4,10 @@ export default {
 	90: 'bsc:textures-vol01',
 	97: 'cleanitol',
 	101: 'bsc:texturepack-cycledogg-vol01',
+	167: [
+		'bsc:mega-props-jrj-vol04',
+  		'bsc:mega-props-jrj-vol05',
+	],
 	272: 'bsc:mega-props-snm-vol01',
 	283: 'girafe:hedges',
 	294: 'girafe:common-spruces',
@@ -30,6 +34,7 @@ export default {
 	851: 'nam',
 	925: 'sfbt:kenworth-rail-catenaries-props',
 	1180: 'bsc:mega-props-cp-vol01',
+	1181: 'sc4jeeks:parking-textures-vol01',
 	1212: 'bsc:mega-props-kevdan25-vol01',
 	1263: 'bsc:mega-props-jes-vol05',
 	1416: 'bsc:mega-props-jes-vol06',
@@ -72,6 +77,7 @@ export default {
 	2945: 'girafe:common-spruces',
 	2962: 'girafe:poppies',
 	2964: 'girafe:narcissus',
+	2979: 'vip:vnaoned-props-pack-vol01',
 	2980: 'girafe:sparaxis',
 	2983: 'girafe:cypresses',
 	2995: 'girafe:beeches',
@@ -99,7 +105,10 @@ export default {
 	3586: 'simmer2:essentials',
 	3640: 'simmer2:mega-prop-pack-vol1',
 	3645: 'simmer2:mega-prop-pack-vol2',
+	3671: 'mattb325:house-15028r',
 	3672: 'bsc:bat-props-mattb325-vol04',
+	3709: 'mattb325:urban-civics-prop-pack-vol01',
+	3710: 'mattb325:urban-civics-prop-pack-vol01',
 	3787: 'simmer2:mega-prop-pack-vol3',
 	3878: 'simmer2:water-reservoir',
 	3885: 'simmer2:mega-prop-pack-vol4',

scripts/traverse-yaml.js

@@ -30,7 +30,7 @@ export default async function traverse(patterns, fn = () => {}, opts = {}) {
 		for (let doc of parseAllDocuments(contents)) {
 			let raw = contents.slice(doc.range[0], doc.range[1]);
 			let json = doc.toJSON();
-			let result = await fn(json, doc, raw);
+			let result = await fn(json, doc, raw, file);
 			if (result) {
 				if (result instanceof Document) {
 					docs.push(result);

src/yaml/mattb325/27672-cell-phone-tower.yaml

@@ -0,0 +1,38 @@
+group: mattb325
+name: cell-phone-tower
+version: "1.0"
+subfolder: 500-utilities
+info:
+  summary: Cell Phone Tower
+  description: |-
+    A small cell phone tower similar to many seen throughout the world.
+
+    This was based on a request from the Simtropolis BAT request thread by LDW2483.
+
+    In game this is presented as a small power plant (it is therefore found in the power menu) which provides employment for one technician.
+
+    As a result it needs road access.
+
+    I find this is actually extremely handy to provide a small boost to the power grid within an extablished city tile as well as giving desirable civic jobs.
+
+    This Cell Phone Tower sits on a 1x1 lot and has values very similar to the in-game wind power plant: meaning primarily that it does not generate pollution, does not explode and does not have a detrimental effect on surrounding areas. Plop, maintenance, bulldoze and power output are identical to the in-game wind power plant.
+
+    A little white flashing light on the top of the mast lets you know that it is working.
+
+    This file has one dependency: [BSC Essentials](https://www.sc4devotion.com/csxlex/lex_filedesc.php?lotGET=443)
+  author: mattb325
+  website: https://community.simtropolis.com/files/file/27672-mattb325-cell-phone-tower/
+  images:
+    - https://www.simtropolis.com/objects/screens/monthly_06_2012/80d69dc5905714abe3b00aba0aace22b-.jpg
+    - https://www.simtropolis.com/objects/screens/monthly_06_2012/34faeb48b82fd23c4bae7dbc09f3f32e-.jpg
+    - https://www.simtropolis.com/objects/screens/monthly_06_2012/05b62265084ffe9f3bc3ab2af469b6f8-.jpg
+dependencies:
+  - bsc:essentials
+assets:
+  - assetId: mattb325-cell-phone-tower
+
+---
+assetId: mattb325-cell-phone-tower
+version: "1.0"
+lastModified: "2012-06-03T09:27:30Z"
+url: https://community.simtropolis.com/files/file/27672-mattb325-cell-phone-tower/?do=download&r=101763

src/yaml/mattb325/27975-old-market-hall.yaml

@@ -0,0 +1,68 @@
+group: mattb325
+name: old-market-hall
+version: "1.0"
+subfolder: 360-landmark
+info:
+  summary: Old Market Hall
+  description: |-
+    **The Old Market Hall, by Mattb325.**
+
+    In major cities everywhere, the Market Hall has always been the centre of community focus. Originally used by farmers and market gardeners to sell produce, they quickly became a merchant hub where all manner of meat, fish, dairy and other wares could be bought and sold.
+
+    In cities where Market Halls were not purpose built, many old industrial warehouses have been re-purposed as these are particularly suitable given their central location, cavernous spaces and abundant natural light. In real life, Market Halls have a great impact to property values nearby: they offer the health benefits of being able to buy very fresh, organic and in-season food from a variety of local growers.
+
+    They provide large scale, basic employment and the larger Market Halls are almost always a tourist attraction.
+
+    Maxis provided an in-game farmers market designed for smaller agricultural towns. The farmers market provides health benefits and lifts demand caps while providing some R$ jobs. However, Maxis, however, did not provide anything for cities and large towns which was quite a shame.
+
+    So this is where the 'Old Market Hall' picks up where Maxis left off by providing slightly greater health benefits, 250 R$ *and* 100 R$$ jobs, helping to lift demand caps, raising the mayors rating and providing a tourist building plus a landmark effect. To provide all these benefits and jobs to the city costs $50 per month from the Parks & Rec coffers.
+
+    The Old Market Hall building is offered as a reward (which is available at anytime). It **does not** over-write the in game Farmers Market and sits on a 4x5 lot. It is of course found in the rewards menu.
+
+    The building itself is inspired by a building in downtown Roanoke, Virginia which was constructed in 1922.
+
+    **STATS:**
+
+    Plop Cost: $31,700
+
+    Bulldoze Cost: $15,000
+
+    Power Consumed: 50Mwh
+
+    Water Consumed: 28 Gallons
+
+    Pollution: 1 (air) / 1 (water) / 11 (garbage)
+
+    Radius: 1 / 2 / 0
+
+    Occupant Groups: Reward, Tourist, Health - Other
+
+    **DEPENDENCIES:**
+
+    BSC BAT Props Mattb325 Vol02 ([http://sc4devotion.c...php?lotGET=2383](https://www.sc4devotion.com/csxlex/lex_filedesc.php?lotGET=2383))
+
+    To install, simply unzip this file into your plugins folder.
+
+    Thankyou, and enjoy.
+
+    **Mac Users**
+
+    The original lot does not work with the Mac version of the game. If you use a Mac, after installation, replace the original .SC4Lot file with the one contained within "Mac Fix.zip".
+  author: mattb325
+  website: https://community.simtropolis.com/files/file/27975-old-market-hall/
+  images:
+    - https://www.simtropolis.com/objects/screens/monthly_08_2012/a4bf119baba87ee8e82860ee756194d9-mkthall.jpg
+    - https://www.simtropolis.com/objects/screens/monthly_08_2012/29e0655057d5723317fd36dc1725e5fa-mkthall1.jpg
+    - https://www.simtropolis.com/objects/screens/monthly_08_2012/ffe8b6f426e8079595c65edd3c45f18c-mkthall2.jpg
+    - https://www.simtropolis.com/objects/screens/monthly_08_2012/d1d726ec8b67cd9a33b4a27f6bfb632a-mkthall3.jpg
+    - https://www.simtropolis.com/objects/screens/monthly_08_2012/f43e963a69f391ebfcef738e52c9dfbc-mkthall4.jpg
+dependencies:
+  - bsc:bat-props-mattb325-vol02
+assets:
+  - assetId: mattb325-old-market-hall
+
+---
+assetId: mattb325-old-market-hall
+version: "1.0"
+lastModified: "2023-09-08T01:54:01Z"
+url: https://community.simtropolis.com/files/file/27975-old-market-hall/?do=download&r=197589