Skip to content

Commit

Permalink
Implement fully custom subnet layouts
Browse files Browse the repository at this point in the history
  • Loading branch information
@danielrbradley
danielrbradley committed Nov 16, 2023
1 parent fdac7b8 commit 3b889da
Show file tree
Hide file tree
Showing 11 changed files with 455 additions and 37 deletions.
142 changes: 141 additions & 1 deletion awsx/ec2/subnetDistributorNew.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,13 @@

import fc from "fast-check";
import { SubnetSpecInputs, SubnetTypeInputs } from "../schema-types";
import { defaultSubnetInputs, getSubnetSpecs, nextNetmask } from "./subnetDistributorNew";
import {
defaultSubnetInputs,
getSubnetSpecs,
nextNetmask,
validSubnetSizes,
validateAndNormalizeSubnetInputs,
} from "./subnetDistributorNew";
import { Netmask } from "netmask";
import { getOverlappingSubnets, validateNoGaps, validateSubnets } from "./vpc";
import { getSubnetSpecsLegacy } from "./subnetDistributorLegacy";
Expand Down Expand Up @@ -267,3 +273,137 @@ describe("validating exact layouts", () => {
});
});
});

describe("explicit subnet layouts", () => {
it("should produce specified subnets", () => {
const result = getSubnetSpecs(
"vpcName",
"10.0.0.0/16",
["us-east-1a", "us-east-1b"],
[
{ type: "Public", cidrBlocks: ["10.0.0.0/18", "10.0.64.0/19"] },
{ type: "Private", cidrBlocks: ["10.0.96.0/19", "10.0.128.0/20"] },
],
);
expect(result).toEqual([
{
azName: "us-east-1a",
cidrBlock: "10.0.0.0/18",
subnetName: "vpcName-public-1",
type: "Public",
},
{
azName: "us-east-1a",
cidrBlock: "10.0.96.0/19",
subnetName: "vpcName-private-1",
type: "Private",
},
{
azName: "us-east-1b",
cidrBlock: "10.0.64.0/19",
subnetName: "vpcName-public-2",
type: "Public",
},
{
azName: "us-east-1b",
cidrBlock: "10.0.128.0/20",
subnetName: "vpcName-private-2",
type: "Private",
},
]);
});
});

describe("valid subnet sizes", () => {
const sizes = validSubnetSizes;
expect(sizes.length).toBe(31);
for (let index = 0; index < sizes.length; index++) {
const size = sizes[index];
// Index is also the netmask
expect(size).toEqual(4294967296 / 2 ** index);
}
});

describe("validating and normalizing inputs", () => {
it("detects invalid sizes", () => {
expect(() => validateAndNormalizeSubnetInputs([{ type: "Public", size: 100 }], 1)).toThrowError(
"The following subnet sizes are invalid: 100. Valid sizes are: ",
);
});
it("detects mismatched size and netmask", () => {
expect(() =>
validateAndNormalizeSubnetInputs([{ type: "Public", size: 4096, cidrMask: 21 }], 1),
).toThrowError("Subnet size 4096 does not match the expected size for a /21 subnet (2048).");
});
it("allows size only", () => {
const result = validateAndNormalizeSubnetInputs([{ type: "Public", size: 1024 }], 1);
expect(result!.normalizedSpecs).toEqual([{ type: "Public", size: 1024, cidrMask: 22 }]);
});
it("allows cidrMask only", () => {
const result = validateAndNormalizeSubnetInputs([{ type: "Public", cidrMask: 23 }], 1);
expect(result!.normalizedSpecs).toEqual([{ type: "Public", size: 512, cidrMask: 23 }]);
});
it("allows cidrMask and size when matching", () => {
const result = validateAndNormalizeSubnetInputs(
[{ type: "Public", cidrMask: 24, size: 256 }],
1,
);
expect(result!.normalizedSpecs).toEqual([{ type: "Public", size: 256, cidrMask: 24 }]);
});
describe("explicit layouts", () => {
it("detects block count mismatching AZ count", () => {
expect(() =>
validateAndNormalizeSubnetInputs([{ type: "Public", cidrBlocks: ["10.0.0.0/16"] }], 2),
).toThrowError(
"The number of CIDR blocks in subnetSpecs[0] must match the number of availability zones (2).",
);
});
it("detects partially specified blocks", () => {
expect(() =>
validateAndNormalizeSubnetInputs(
[{ type: "Public", cidrBlocks: ["10.0.0.0/16"] }, { type: "Private" }],
1,
),
).toThrowError(
"If any subnet spec has explicit cidrBlocks, all subnets must have explicit cidrBlocks.",
);
});
it("detects cidr blocks with mismatched netmask", () => {
expect(() =>
validateAndNormalizeSubnetInputs(
[{ type: "Public", cidrBlocks: ["10.0.0.0/16"], cidrMask: 17 }],
1,
),
).toThrowError(
"The cidrMask in subnetSpecs[0] must match all cidrBlocks or be left undefined.",
);
});
it("detects cidr blocks with mismatched netmask", () => {
expect(() =>
validateAndNormalizeSubnetInputs(
[{ type: "Public", cidrBlocks: ["10.0.0.0/16"], size: 1024 }],
1,
),
).toThrowError("The size in subnetSpecs[0] must match all cidrBlocks or be left undefined.");
});
it("allows all argument to be in agreement", () => {
validateAndNormalizeSubnetInputs(
[
{
type: "Public",
cidrBlocks: ["10.0.0.0/20", "10.0.16.0/20"],
size: 4096,
cidrMask: 20,
},
{
type: "Public",
cidrBlocks: ["10.0.32.0/21", "10.0.40.0/21"],
size: 2048,
cidrMask: 21,
},
],
2,
);
});
});
});
139 changes: 124 additions & 15 deletions awsx/ec2/subnetDistributorNew.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,23 +61,28 @@ export function getSubnetSpecs(

let currentAzNetmask = new Netmask(`${vpcNetmask.base}/${azBitmask}`);
const subnets: SubnetSpec[] = [];
let azNum = 1;
for (const azName of azNames) {
for (let azIndex = 0; azIndex < azNames.length; azIndex++) {
const azName = azNames[azIndex];
const azNum = azIndex + 1;
let currentSubnetNetmask: Netmask | undefined;
for (const subnetSpec of subnetSpecs) {
if (currentSubnetNetmask === undefined) {
currentSubnetNetmask = new Netmask(
currentAzNetmask.base,
subnetSpec.cidrMask ?? defaultSubnetBitmask,
);
} else {
currentSubnetNetmask = nextNetmask(
currentSubnetNetmask,
subnetSpec.cidrMask ?? defaultSubnetBitmask,
);
let subnetCidr = subnetSpec.cidrBlocks?.[azIndex];
if (subnetCidr === undefined) {
if (currentSubnetNetmask === undefined) {
currentSubnetNetmask = new Netmask(
currentAzNetmask.base,
subnetSpec.cidrMask ?? defaultSubnetBitmask,
);
} else {
currentSubnetNetmask = nextNetmask(
currentSubnetNetmask,
subnetSpec.cidrMask ?? defaultSubnetBitmask,
);
}
subnetCidr = currentSubnetNetmask.toString();
}
const subnetCidr = currentSubnetNetmask.toString();
const subnetName = `${vpcName}-${subnetSpec.type.toLowerCase()}-${azNum}`;
const specName = subnetSpec.name ?? subnetSpec.type.toLowerCase();
const subnetName = `${vpcName}-${specName}-${azNum}`;
subnets.push({
cidrBlock: subnetCidr,
type: subnetSpec.type,
Expand All @@ -87,7 +92,6 @@ export function getSubnetSpecs(
}

currentAzNetmask = currentAzNetmask.next();
azNum++;
}

return subnets;
Expand Down Expand Up @@ -149,3 +153,108 @@ function nextPow2(n: number): number {

return n + 1;
}

/* Ensure all inputs are consistent and fill in missing values with defaults
* Ensure any specified, netmask, size or blocks are in agreement.
*/
export function validateAndNormalizeSubnetInputs(
subnetArgs: SubnetSpecInputs[] | undefined,
availabilityZoneCount: number,
): { normalizedSpecs: SubnetSpecInputs[]; isExplicitLayout: boolean } | undefined {
if (subnetArgs === undefined) {
return undefined;
}

const issues: string[] = [];

// All sizes must be valid.
const invalidSizes = subnetArgs.filter(
(spec) => spec.size !== undefined && !validSubnetSizes.includes(spec.size),
);
if (invalidSizes.length > 0) {
issues.push(
`The following subnet sizes are invalid: ${invalidSizes
.map((spec) => spec.size)
.join(", ")}. Valid sizes are: ${validSubnetSizes.join(", ")}.`,
);
}

const hasExplicitLayouts = subnetArgs.some((subnet) => subnet.cidrBlocks !== undefined);
if (hasExplicitLayouts) {
// If any subnet spec has explicit cidrBlocks, all subnets must have explicit cidrBlocks.
const hasMissingExplicitLayouts = subnetArgs.some((subnet) => subnet.cidrBlocks === undefined);
if (hasMissingExplicitLayouts) {
issues.push(
"If any subnet spec has explicit cidrBlocks, all subnets must have explicit cidrBlocks.",
);
}

// Number of cidrBlocks must match the number of availability zones.
for (let specIndex = 0; specIndex < subnetArgs.length; specIndex++) {
const spec = subnetArgs[specIndex];
if (spec.cidrBlocks !== undefined && spec.cidrBlocks.length != availabilityZoneCount) {
issues.push(
`The number of CIDR blocks in subnetSpecs[${specIndex}] must match the number of availability zones (${availabilityZoneCount}).`,
);
}
}

// Any size or cidrMask must be in agreement with the cidrBlocks.
for (let specIndex = 0; specIndex < subnetArgs.length; specIndex++) {
const spec = subnetArgs[specIndex];
if (spec.cidrBlocks === undefined) {
continue;
}
const blockMasks = spec.cidrBlocks!.map((b) => new Netmask(b));
if (spec.cidrMask !== undefined) {
if (!blockMasks.every((b) => b.bitmask === spec.cidrMask)) {
issues.push(
`The cidrMask in subnetSpecs[${specIndex}] must match all cidrBlocks or be left undefined.`,
);
}
}
if (spec.size !== undefined) {
if (!blockMasks.every((b) => b.size === spec.size!)) {
issues.push(
`The size in subnetSpecs[${specIndex}] must match all cidrBlocks or be left undefined.`,
);
}
}
}

if (issues.length === 0) {
return { normalizedSpecs: subnetArgs, isExplicitLayout: true };
}
} else {
const normalizedSpecs = subnetArgs.map((spec) => {
// Ensure size and cidrMask are in agreement.
const cidrMask =
spec.cidrMask ?? (spec.size ? validSubnetSizes.indexOf(spec.size!) : undefined);
const expectedSize = cidrMask ? validSubnetSizes[cidrMask] : undefined;
if (spec.size !== undefined && expectedSize !== undefined && expectedSize !== spec.size) {
issues.push(
`Subnet size ${spec.size} does not match the expected size for a /${cidrMask} subnet (${expectedSize}).`,
);
}
// Set both cidrMask and size to the resolved values, if either was provided.
return {
...spec,
cidrMask: cidrMask,
size: expectedSize,
};
});

if (issues.length === 0) {
return { normalizedSpecs, isExplicitLayout: false };
}
}

throw new Error(`Invalid subnet specifications:\n - ${issues.join("\n - ")}`);
}

// The index of the array is the corresponding netmask.
export const validSubnetSizes: readonly number[] = [
4294967296, 2147483648, 1073741824, 536870912, 268435456, 134217728, 67108864, 33554432, 16777216,
8388608, 4194304, 2097152, 1048576, 524288, 262144, 131072, 65536, 32768, 16384, 8192, 4096, 2048,
1024, 512, 256, 128, 64, 32, 16, 8, 4,
];
16 changes: 11 additions & 5 deletions awsx/ec2/vpc.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import * as aws from "@pulumi/aws";
import * as pulumi from "@pulumi/pulumi";
import * as schema from "../schema-types";
import { getSubnetSpecsLegacy, SubnetSpec } from "./subnetDistributorLegacy";
import { getSubnetSpecs } from "./subnetDistributorNew";
import { getSubnetSpecs, validateAndNormalizeSubnetInputs } from "./subnetDistributorNew";
import { Netmask } from "netmask";

interface VpcData {
Expand Down Expand Up @@ -82,11 +82,17 @@ export class Vpc extends schema.Vpc<VpcData> {
const cidrBlock = args.cidrBlock ?? "10.0.0.0/16";

const subnetStrategy = args.subnetStrategy ?? "Legacy";
const parsedSpecs = validateAndNormalizeSubnetInputs(
args.subnetSpecs,
availabilityZones.length,
);

const subnetSpecs =
subnetStrategy === "Legacy"
? getSubnetSpecsLegacy(name, cidrBlock, availabilityZones, args.subnetSpecs)
: getSubnetSpecs(name, cidrBlock, availabilityZones, args.subnetSpecs);
const subnetSpecs = (() => {
if (parsedSpecs?.isExplicitLayout || subnetStrategy !== "Legacy") {
return getSubnetSpecs(name, cidrBlock, availabilityZones, parsedSpecs?.normalizedSpecs);
}
return getSubnetSpecsLegacy(name, cidrBlock, availabilityZones, parsedSpecs?.normalizedSpecs);
})();

validateSubnets(subnetSpecs, getOverlappingSubnets);

Expand Down
4 changes: 4 additions & 0 deletions awsx/schema-types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -572,14 +572,18 @@ export type NatGatewayStrategyOutputs = "None" | "Single" | "OnePerAz";
export type SubnetAllocationStrategyInputs = "Legacy" | "Auto" | "Exact";
export type SubnetAllocationStrategyOutputs = "Legacy" | "Auto" | "Exact";
export interface SubnetSpecInputs {
readonly cidrBlocks?: string[];
readonly cidrMask?: number;
readonly name?: string;
readonly size?: number;
readonly tags?: pulumi.Input<Record<string, pulumi.Input<string>>>;
readonly type: SubnetTypeInputs;
}
export interface SubnetSpecOutputs {
readonly cidrBlocks?: string[];
readonly cidrMask?: number;
readonly name?: string;
readonly size?: number;
readonly tags?: pulumi.Output<Record<string, string>>;
readonly type: SubnetTypeOutputs;
}
Expand Down
16 changes: 15 additions & 1 deletion awsx/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -584,16 +584,30 @@
"awsx:ec2:SubnetSpec": {
"description": "Configuration for a VPC subnet.",
"properties": {
"cidrBlocks": {
"type": "array",
"items": {
"type": "string",
"plain": true
},
"plain": true,
"description": "An optional list of CIDR blocks to assign to the subnet spec for each AZ. If specified, the count must match the number of AZs being used for the VPC, and must also be specified for all other subnet specs."
},
"cidrMask": {
"type": "integer",
"plain": true,
"description": "The bitmask for the subnet's CIDR block. The default value is set based on an even distribution of available space from the VPC's CIDR block after being divided evenly by availability zone."
"description": "The netmask for the subnet's CIDR block. This is optional, the default value is inferred from the `cidrMask`, `cidrBlocks` or based on an even distribution of available space from the VPC's CIDR block after being divided evenly by availability zone."
},
"name": {
"type": "string",
"plain": true,
"description": "The subnet's name. Will be templated upon creation."
},
"size": {
"type": "integer",
"plain": true,
"description": "Optional size of the subnet's CIDR block - the number of hosts. This value must be a power of 2 (e.g. 256, 512, 1024, etc.). This is optional, the default value is inferred from the `cidrMask`, `cidrBlocks` or based on an even distribution of available space from the VPC's CIDR block after being divided evenly by availability zone."
},
"tags": {
"type": "object",
"additionalProperties": {
Expand Down
Loading

0 comments on commit 3b889da

Please sign in to comment.