Skip to content

Commit

Permalink
improve module resolution and type hints
Browse files Browse the repository at this point in the history
  • Loading branch information
@Matchlighter
Matchlighter committed Feb 21, 2024
1 parent 6270e9f commit 7528cf1
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 70 deletions.
10 changes: 5 additions & 5 deletions src/common/generate_tsconfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ import { TsConfigJson } from "type-fest";
import fse = require('fs-extra');

import { merger } from "@matchlighter/common_library/data/config";
import { Hypervisor } from "../hypervisor/hypervisor";
import { TD_DEVELOPER_MODE, TYPEDAEMON_PATH } from "./util";
import { Configuration } from "../hypervisor/config";
import { Hypervisor } from "../hypervisor/hypervisor";
import { PATH_ALIASES } from "../hypervisor/vm";
import { TYPEDAEMON_PATH } from "./util";

const tsconfigMerger = merger<TsConfigJson>({
compilerOptions: merger({
Expand All @@ -27,17 +27,17 @@ const tsconfigMerger = merger<TsConfigJson>({
})

export async function saveGeneratedTsconfig(hv: Hypervisor) {
const typedaemon_dir = TD_DEVELOPER_MODE ? path.relative(hv.operations_directory, TYPEDAEMON_PATH) : "./node_modules/typedaemon/dist";
const typedaemon_dir = path.relative(hv.operations_directory, TYPEDAEMON_PATH);
const cfg = hv.currentConfig as Configuration;

const paths = {}

for (let [k, v] of Object.entries(PATH_ALIASES)) {
paths[k] = [v.replace("@TYPEDAEMON", typedaemon_dir)]
paths[k] = [v.replace("@TYPEDAEMON", typedaemon_dir).replace("@SYS_NODE_MODULES", path.join(typedaemon_dir, '..'))]
}

Object.assign(paths, {
"*": ["./node_modules/*"],
// "*": ["./node_modules/*"],
})

let tscfg: TsConfigJson = {
Expand Down
38 changes: 3 additions & 35 deletions src/hypervisor/application_instance.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { BaseInstance, InstanceLogConfig } from './managed_apps';
import { RequireRestart, configChangeHandler } from './managed_config_events';
import { installDependencies } from './packages';
import { PersistentStorage } from './persistent_storage';
import { createApplicationVM } from './vm';
import { createApplicationVM, determineModuleContext } from './vm';

export interface ApplicationMetadata {
applicationClass?: typeof Application;
Expand Down Expand Up @@ -78,44 +78,12 @@ export class ApplicationInstance extends BaseInstance<AppConfiguration, Applicat
}
}

includedFileScope(file: string) {
if (!file) return "sandbox";

// Make sure TypeDaemon stuff always runs on the Host
if (file.includes(TYPEDAEMON_PATH)) {
return "host";
}

// _Any_ files in the app directory should run in the Sandbox
if (file.includes(this.source_directory)) {
return "sandbox";
}

// In fact, any files in the applications directory should run in the Sandbox
if (file.includes(path.resolve(this.hypervisor.working_directory, 'applications'))) {
return "sandbox";
}

// Any Lite-App Environments should run in the Sandbox
if (file.includes(path.resolve(this.shared_operating_directory))) {
return "sandbox";
}

// TODO If hosted_module, "host"
// TODO If global dependency, "host"

if (file.match(/node_modules/)) {
return "host"
}

return "sandbox";
}

private watchedDependencies = new Set<string>();
markFileDependency(file: string, calling_module?: string) {
if (!this.options.watch?.source) return;
if (this.includedFileScope(file) != "sandbox") return;
if (determineModuleContext(file) != "sandbox") return;
if (file.includes("/node_modules/")) return;
if (file.includes(TYPEDAEMON_PATH) && !file.includes(this.hypervisor.working_directory)) return;

// Don't watch the new file if it was somehow hopped to
// if (calling_module && !this.watchedDependencies.has(calling_module)) return;
Expand Down
93 changes: 65 additions & 28 deletions src/hypervisor/vm.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,11 @@ export const PATH_ALIASES = {
"@td/ha": "@TYPEDAEMON/plugins/home_assistant/api",
"@td/mqtt": "@TYPEDAEMON/plugins/mqtt/api",
"@td/http": "@TYPEDAEMON/plugins/http/api",
"@td/*": "@TYPEDAEMON/*",
"@td/util/*": "@TYPEDAEMON/runtime/util/*",
"typedaemon/*": "@TYPEDAEMON/*",
// "@td/*": "@TYPEDAEMON/*",
"mobx": "@SYS_NODE_MODULES/mobx",
"axios": "@SYS_NODE_MODULES/axios",
}

export async function loadPathMaps(entrypoint: string) {
Expand Down Expand Up @@ -74,12 +78,63 @@ export async function loadPathMaps(entrypoint: string) {
}

for (let [k, v] of Object.entries(PATH_ALIASES)) {
path_maps[k] = [v.replace("@TYPEDAEMON", TYPEDAEMON_PATH)]
path_maps[k] = [v.replace("@TYPEDAEMON", TYPEDAEMON_PATH).replace("@SYS_NODE_MODULES", TD_MODULES_PATH)]
}

return path_maps;
}

const SYSTEM_MODULES: (string | RegExp)[] = [
'mobx',
'typedaemon',
'axios',
]
const EXTENSIONS = ["js", "ts", "jsx", "tsx"];

function isSystemModule(x: string) {
let matched = false;
for (let sm of SYSTEM_MODULES) {
if (sm instanceof RegExp) {
if (x.match(sm)) {
matched = true;
break;
};
} else {
if (x == sm || x.startsWith(sm + '/')) {
matched = true;
break;
}
}
}
return matched;
}

export function determineModuleContext(filename: string) {
if (filename.startsWith(TYPEDAEMON_PATH)) {
let sys_rel = path.relative(TYPEDAEMON_PATH, filename);

if (!sys_rel.startsWith("node_modules/")) { // In dev, node_modules are a subdir of TD
if (sys_rel.match(/^(\w+\/)?runtime\/util\//)) {
return "sandbox";
}

return "host";
}
}

if (filename.startsWith(TD_MODULES_PATH)) {
let sys_rel = path.relative(TD_MODULES_PATH, filename);

if (isSystemModule(sys_rel)) {
return "host";
}
}

// TODO If hosted_module, "host"

return "sandbox";
}

export async function createApplicationVM(app: ApplicationInstance) {
const consoleMethods: Console = {} as any;
for (let m of CONSOLE_METHODS) {
Expand All @@ -94,13 +149,6 @@ export async function createApplicationVM(app: ApplicationInstance) {
await loadPathMaps(app.entrypoint),
);

const SYSTEM_MODULES: (string | RegExp)[] = [
'mobx',
'typedaemon',
'axios',
]
const EXTENSIONS = ["js", "ts", "jsx", "tsx"]

const opModulesPath = path.join(app.shared_operating_directory, "node_modules");

const nativeTimers = new Set<any>();
Expand Down Expand Up @@ -157,6 +205,7 @@ export async function createApplicationVM(app: ApplicationInstance) {
filename,
})

// TODO Fix memory leak
registerSourceMap(filename, result.map);

return result.code;
Expand All @@ -167,6 +216,8 @@ export async function createApplicationVM(app: ApplicationInstance) {
context: "host",
external: true,
customRequire(id) {
logMessage("debug", `Requiring host module '${id}'`)

// Supply a patched MobX that will automatically add Reaction disposers to the cleanups
if (id?.includes("node_modules/mobx/dist/")) {
return appmobx;
Expand All @@ -184,7 +235,7 @@ export async function createApplicationVM(app: ApplicationInstance) {
const vmResolver: VMInternalResolver = vm['_resolver'];

patch(vmResolver, 'pathContext', original => function (filename, filetype) {
return app.includedFileScope(filename);
return determineModuleContext(filename);
});

// Can the specified file be loaded by the VM
Expand All @@ -200,7 +251,8 @@ export async function createApplicationVM(app: ApplicationInstance) {

const resolvedTo: string = original.call(this, calling_module, requested_module, opts, extension_handlers, direct)
// resolvedTo should be either an absolute path or an internal module. Don't try to watch an internal
if (resolvedTo.startsWith('/')) {
// TODO Watch client node_modules in a better way (eg detect re-installation)
if (resolvedTo.startsWith('/') && !resolvedTo.includes(TD_MODULES_PATH)) {
app.markFileDependency(resolvedTo, calling_module.filename);
}
logMessage("debug", `Resolved module '${requested_module.toString()}' from ${calling_module.filename} to ${resolvedTo}`);
Expand All @@ -210,7 +262,7 @@ export async function createApplicationVM(app: ApplicationInstance) {
const system_modules_folders = vmResolver.genLookupPaths(TD_MODULES_PATH)

patch(vmResolver, 'genLookupPaths', original => function (curPath) {
const paths = original(curPath);
let paths = original(curPath);

// Inject the install location of the app's dependencies
if (!paths.includes(opModulesPath)) paths.unshift(opModulesPath)
Expand All @@ -220,22 +272,7 @@ export async function createApplicationVM(app: ApplicationInstance) {

patch(vmResolver, 'loadNodeModules', original => function (x, dirs, extList) {
// Ensure typedaemon, MobX, etc. will _never_ load from an application's node_modules.
let matched = false;
for (let sm of SYSTEM_MODULES) {
if (sm instanceof RegExp) {
if (x.match(sm)) {
matched = true;
break;
};
} else {
if (x == sm || x.startsWith(sm + '/')) {
matched = true;
break;
}
}
}

if (matched) {
if (isSystemModule(x)) {
const resolv = original(x, system_modules_folders, extList);
if (resolv) return resolv;
logMessage("warn", `${x} was determined to be a system module, but it could not be found in the system. Falling back to app resolution.`)
Expand Down
2 changes: 0 additions & 2 deletions src/runtime/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ export { resumable } from './resumable';
export * as schedule from './schedule';
export { sleep, sleep_until } from "./sleep";

export * as func from "./func";

/**
* Retrieve a handle to an application instance for the given Application id
*/
Expand Down
0 src/runtime/func.ts → src/runtime/util/func.ts
View file
File renamed without changes.

0 comments on commit 7528cf1

Please sign in to comment.