From 37da859b3380770665d355bcea8486333f1ef3a0 Mon Sep 17 00:00:00 2001 From: apfadler Date: Tue, 7 Jun 2016 20:03:53 +0200 Subject: [PATCH] [POC-4] integration of qlobject --- .../CachedFileSystemRepository.java | 332 ++++++++++++++++++ dist/config/java.util.logging.properties | 4 +- dist/config/quil-client.xml | 31 ++ dist/config/quil-common.xml | 142 ++++++++ dist/config/quil-server.xml | 29 ++ dist/sampledata/Data.Trades.csv | 7 - .../{ => FlatFiles}/Data.Market.csv | 2 +- dist/sampledata/FlatFiles/Data.Trades.csv | 7 + .../Script.Example.General.scala} | 2 +- .../QLObjects/Controller.Example.scala | 32 ++ .../Parametrization/Curves.json} | 9 - .../QLObjects/Parametrization/Mappings.json | 11 + .../QLObjects/Task.QuantlibObjects.json | 24 ++ .../{ => QLObjects/Trades}/trades.json | 0 ...Script.QuantlibScript.ExamplePricer.scala} | 5 +- .../Task.PriceTrades.QLScript.json | 0 .../Task.QuantLibScript.json | 0 .../Task.PriceSingleTrade.json | 0 .../Task.PriceTrades.json | 0 .../Template.Trade.Swap.xml | 0 dist/sampledata/Task.QuantlibObjects.json | 20 -- .../QuantLibScriptInterpreter.java | 10 +- .../java/org/quil/server/ResultsCache.java | 0 .../org/quil/server/Tasks/PricePortfolio.java | 0 .../org/quil/server/Tasks/PriceTrade.java | 0 .../org/quil/server/Tasks/ResultItem.java | 0 .../org/quil/server/Tasks/ScriptedTask.java | 0 .../main/java/org/quil/server/Tasks/Task.java | 16 +- qlobjects/src/main/main.iml | 6 + .../QuantLibObjects/Controller.scala | 8 + .../QLObjectsInterpreter.scala | 139 +++++++- .../Tasks/RunQLObjectsApplication.scala | 54 +++ .../main/java/org/quil/server/ClusterAPI.java | 5 - .../main/java/org/quil/server/QuilServer.java | 26 +- .../java/org/quil/server/RepositoryAPI.java | 26 +- .../main/java/org/quil/server/TaskAPI.java | 2 +- webapp/src/main/webapp/js/controllers.js | 19 +- 37 files changed, 873 insertions(+), 95 deletions(-) create mode 100644 common/src/main/java/org/quil/repository/CachedFileSystemRepository.java create mode 100644 dist/config/quil-client.xml create mode 100644 dist/config/quil-common.xml create mode 100644 dist/config/quil-server.xml delete mode 100644 dist/sampledata/Data.Trades.csv rename dist/sampledata/{ => FlatFiles}/Data.Market.csv (65%) create mode 100644 dist/sampledata/FlatFiles/Data.Trades.csv rename dist/sampledata/{Script.Example.scala => GeneralScripts/Script.Example.General.scala} (96%) create mode 100644 dist/sampledata/QLObjects/Controller.Example.scala rename dist/sampledata/{config.json => QLObjects/Parametrization/Curves.json} (96%) create mode 100644 dist/sampledata/QLObjects/Parametrization/Mappings.json create mode 100644 dist/sampledata/QLObjects/Task.QuantlibObjects.json rename dist/sampledata/{ => QLObjects/Trades}/trades.json (100%) rename dist/sampledata/{Script.Pricer.scala => QLScriptPricer/Script.QuantlibScript.ExamplePricer.scala} (91%) rename dist/sampledata/{ => QLScriptPricer}/Task.PriceTrades.QLScript.json (100%) rename dist/sampledata/{ => QLScriptPricer}/Task.QuantLibScript.json (100%) rename dist/sampledata/{ => QLXMLTemplates}/Task.PriceSingleTrade.json (100%) rename dist/sampledata/{ => QLXMLTemplates}/Task.PriceTrades.json (100%) rename dist/sampledata/{ => QLXMLTemplates}/Template.Trade.Swap.xml (100%) delete mode 100644 dist/sampledata/Task.QuantlibObjects.json rename {server => interpreter}/src/main/java/org/quil/server/ResultsCache.java (100%) rename {server => interpreter}/src/main/java/org/quil/server/Tasks/PricePortfolio.java (100%) rename {server => interpreter}/src/main/java/org/quil/server/Tasks/PriceTrade.java (100%) rename {server => interpreter}/src/main/java/org/quil/server/Tasks/ResultItem.java (100%) rename {server => interpreter}/src/main/java/org/quil/server/Tasks/ScriptedTask.java (100%) rename {server => interpreter}/src/main/java/org/quil/server/Tasks/Task.java (98%) create mode 100644 qlobjects/src/main/main.iml create mode 100644 qlobjects/src/main/scala/org/quil/interpreter/QuantLibObjects/Controller.scala create mode 100644 qlobjects/src/main/scala/org/quil/server/Tasks/RunQLObjectsApplication.scala diff --git a/common/src/main/java/org/quil/repository/CachedFileSystemRepository.java b/common/src/main/java/org/quil/repository/CachedFileSystemRepository.java new file mode 100644 index 0000000..1518de6 --- /dev/null +++ b/common/src/main/java/org/quil/repository/CachedFileSystemRepository.java @@ -0,0 +1,332 @@ +package org.quil.repository; + +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteFileSystem; +import org.apache.ignite.Ignition; +import org.apache.ignite.igfs.IgfsException; +import org.apache.ignite.igfs.IgfsInputStream; +import org.apache.ignite.igfs.IgfsPath; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Collection; + +//TODO extract interface + +public class CachedFileSystemRepository { + + static final Logger logger = LoggerFactory.getLogger(CachedFileSystemRepository.class); + + private static CachedFileSystemRepository _instance; + + private String _baseFolder; + + public static CachedFileSystemRepository instance() { + if (_instance == null) + _instance = new CachedFileSystemRepository(); + + return _instance; + } + + public CachedFileSystemRepository() { + + _baseFolder = System.getenv("QUIL_HOME")+"/sampledata/"; + + logger.info("Building IGFS cache for repository."); + initIgfs(null, "/"); + } + + protected void initIgfs(File node, String path) { + + logger.info("initIGFS: " + path); + + Ignite ignite = Ignition.ignite(); + IgniteFileSystem fs = ignite.fileSystem("quil-igfs"); + + if (node == null) { + node = new File(_baseFolder); + } + + if(node.isDirectory()){ + + mkdirs(fs, new IgfsPath(path)); + for(File subNode : node.listFiles()){ + + if (node.getAbsolutePath().equals((new File(_baseFolder)).getAbsolutePath())) + initIgfs(subNode, path + subNode.getName() ); + else + initIgfs(subNode, path + "/" + subNode.getName() ); + } + + } else { + try { + File file = new File(_baseFolder + path); + create(fs, new IgfsPath(path), Files.readAllBytes(file.toPath())); + }catch (IOException e ) { + e.printStackTrace(); + logger.info("IOException occured during file creation in igfs."); + } + } + } + + public void deleteFile(String path) { + + Ignite ignite = Ignition.ignite(); + IgniteFileSystem fs = ignite.fileSystem("quil-igfs"); + + File file = new File(_baseFolder+path); + file.delete(); + + delete(fs, new IgfsPath("/"+path)); + } + + public void putFolder(String path) { + + Ignite ignite = Ignition.ignite(); + IgniteFileSystem fs = ignite.fileSystem("quil-igfs"); + + File file = new File(_baseFolder+path); + file.mkdir(); + + mkdirs(fs, new IgfsPath("/"+path)); + } + + public String getFile(String path) throws IOException { + + Ignite ignite = Ignition.ignite(); + IgniteFileSystem fs = ignite.fileSystem("quil-igfs"); + + return new String(read(fs, new IgfsPath("/"+path))); + } + + + public String putFile(String path, String content) throws FileNotFoundException { + + PrintWriter out; + out = new PrintWriter(_baseFolder+path); + out.print(content); + out.close(); + + Ignite ignite = Ignition.ignite(); + IgniteFileSystem fs = ignite.fileSystem("quil-igfs"); + + try { + create(fs, new IgfsPath("/"+path), content.getBytes()); + } catch (IOException e) { + logger.info("IOException occured during put file."); + } + + return null; + } + + public void genRepositoryObject(File node, JSONObject obj, String path) { + + // TODO Too Specific for js UI + if ( node == null) { + node = new File(_baseFolder); + obj.put("id", "/" ); + obj.put("path", path + "/" ); + obj.put("text", "Root"); + } + else { + obj.put("id", path + "/" + node.getName() ); + obj.put("path", path + "/" + node.getName() ); + obj.put("text", node.getName()); + } + + if(node.isDirectory()){ + + JSONArray children = new JSONArray(); + + String[] subNodes = node.list(); + for(String filename : subNodes){ + JSONObject file = new JSONObject(); + if (node.getAbsolutePath().equals((new File(_baseFolder)).getAbsolutePath())) + genRepositoryObject(new File(node, filename),file, path ); + else + genRepositoryObject(new File(node, filename),file, path + "/" + node.getName() ); + children.add(file); + } + + obj.put("children", children); + obj.put("type", "dir"); + obj.put("icon", "jstree-custom-folder"); + } + else + { + obj.put("type", "file"); + obj.put("icon", "jstree-custom-file"); + } + } + + //TODO different behavior for igfs and normal file system + private static void delete(IgniteFileSystem fs, IgfsPath path) throws IgniteException { + assert fs != null; + assert path != null; + + if (fs.exists(path)) { + boolean isFile = fs.info(path).isFile(); + + try { + fs.delete(path, true); + + System.out.println(); + System.out.println(">>> Deleted " + (isFile ? "file" : "directory") + ": " + path); + } + catch (IgfsException e) { + System.out.println(); + System.out.println(">>> Failed to delete " + (isFile ? "file" : "directory") + " [path=" + path + + ", msg=" + e.getMessage() + ']'); + } + } + else { + System.out.println(); + System.out.println(">>> Won't delete file or directory (doesn't exist): " + path); + } + } + + /** + * Creates directories. + * + * @param fs IGFS. + * @param path Directory path. + * @throws IgniteException In case of error. + */ + private static void mkdirs(IgniteFileSystem fs, IgfsPath path) throws IgniteException { + assert fs != null; + assert path != null; + + try { + fs.mkdirs(path); + + System.out.println(); + System.out.println(">>> Created directory: " + path); + } + catch (IgfsException e) { + System.out.println(); + System.out.println(">>> Failed to create a directory [path=" + path + ", msg=" + e.getMessage() + ']'); + } + + System.out.println(); + } + + /** + * Creates file and writes provided data to it. + * + * @param fs IGFS. + * @param path File path. + * @param data Data. + * @throws IgniteException If file can't be created. + * @throws IOException If data can't be written. + */ + private static void create(IgniteFileSystem fs, IgfsPath path, byte[] data) + throws IgniteException, IOException { + assert fs != null; + assert path != null; + + try (OutputStream out = fs.create(path, true)) { + System.out.println(); + System.out.println(">>> Created file: " + path); + + if (data != null) { + out.write(data); + + System.out.println(); + System.out.println(">>> Wrote data to file: " + path); + } + } + + System.out.println(); + } + + /** + * Opens file and appends provided data to it. + * + * @param fs IGFS. + * @param path File path. + * @param data Data. + * @throws IgniteException If file can't be created. + * @throws IOException If data can't be written. + */ + private static void append(IgniteFileSystem fs, IgfsPath path, byte[] data) throws IgniteException, IOException { + assert fs != null; + assert path != null; + assert data != null; + assert fs.info(path).isFile(); + + try (OutputStream out = fs.append(path, true)) { + System.out.println(); + System.out.println(">>> Opened file: " + path); + + out.write(data); + } + + System.out.println(); + System.out.println(">>> Appended data to file: " + path); + } + + /** + * Opens file and reads it to byte array. + * + * @param fs IgniteFs. + * @param path File path. + * @throws IgniteException If file can't be opened. + * @throws IOException If data can't be read. + */ + private static byte[] read(IgniteFileSystem fs, IgfsPath path) throws IgniteException, IOException { + assert fs != null; + assert path != null; + assert fs.info(path).isFile(); + + byte[] data = new byte[(int)fs.info(path).length()]; + + try (IgfsInputStream in = fs.open(path)) { + in.read(data); + } + + System.out.println(); + System.out.println(">>> Read data from " + path ); + + return data; + } + + /** + * Lists files in directory. + * + * @param fs IGFS. + * @param path Directory path. + * @throws IgniteException In case of error. + */ + private static void list(IgniteFileSystem fs, IgfsPath path) throws IgniteException { + assert fs != null; + assert path != null; + assert fs.info(path).isDirectory(); + + Collection files = fs.listPaths(path); + + if (files.isEmpty()) { + System.out.println(); + System.out.println(">>> No files in directory: " + path); + } + else { + System.out.println(); + System.out.println(">>> List of files in directory: " + path); + + for (IgfsPath f : files) + System.out.println(">>> " + f.name()); + } + + System.out.println(); + } +} \ No newline at end of file diff --git a/dist/config/java.util.logging.properties b/dist/config/java.util.logging.properties index 51ec21f..ba707e3 100644 --- a/dist/config/java.util.logging.properties +++ b/dist/config/java.util.logging.properties @@ -83,8 +83,8 @@ log4j.appender.stdout.layout.ConversionPattern=%t %-5p %c{2} - %m%n log4j.logger.org.quil.server.QuilServer = DEBUG, sse log4j.logger.org.quil.server.Tasks.TaskRunner = info,sse -log4j.logger.org.quil.server.Tasks.PriceTrade = info,sse -log4j.logger.org.quil.server.Tasks.PricePortfolio = info, sse +log4j.logger.PriceTrade = info,sse +log4j.logger.PricePortfolio = info, sse log4j.logger.org.quil.server.Interpreter.MoCoXmlTemplateInterpreter = info, sse #log4j.appender.sse.target=System.out diff --git a/dist/config/quil-client.xml b/dist/config/quil-client.xml new file mode 100644 index 0000000..5256b35 --- /dev/null +++ b/dist/config/quil-client.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/dist/config/quil-common.xml b/dist/config/quil-common.xml new file mode 100644 index 0000000..7b4e2c9 --- /dev/null +++ b/dist/config/quil-common.xml @@ -0,0 +1,142 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 127.0.0.1:47500..47509 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dist/config/quil-server.xml b/dist/config/quil-server.xml new file mode 100644 index 0000000..532e044 --- /dev/null +++ b/dist/config/quil-server.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/dist/sampledata/Data.Trades.csv b/dist/sampledata/Data.Trades.csv deleted file mode 100644 index 9c3c540..0000000 --- a/dist/sampledata/Data.Trades.csv +++ /dev/null @@ -1,7 +0,0 @@ -ID;Notional;Counterparty -1;1000;A -2;2000;A -3;3000;B -4;4000;B -5;5000;C -6;6000;C \ No newline at end of file diff --git a/dist/sampledata/Data.Market.csv b/dist/sampledata/FlatFiles/Data.Market.csv similarity index 65% rename from dist/sampledata/Data.Market.csv rename to dist/sampledata/FlatFiles/Data.Market.csv index 15a3a57..f27befb 100644 --- a/dist/sampledata/Data.Market.csv +++ b/dist/sampledata/FlatFiles/Data.Market.csv @@ -1,3 +1,3 @@ -FX_EUR_USD;1.05 +FX_EUR_USD;1.01 FX_EUR_GBP;0.8 EUR_Swap_1Y;0.08 \ No newline at end of file diff --git a/dist/sampledata/FlatFiles/Data.Trades.csv b/dist/sampledata/FlatFiles/Data.Trades.csv new file mode 100644 index 0000000..467f701 --- /dev/null +++ b/dist/sampledata/FlatFiles/Data.Trades.csv @@ -0,0 +1,7 @@ +ID;Notional;Counterparty +1;1000.00;A +2;2000.00;A +3;3000.00;B +4;4000.00;B +5;5000.00;C +6;6000.00;C \ No newline at end of file diff --git a/dist/sampledata/Script.Example.scala b/dist/sampledata/GeneralScripts/Script.Example.General.scala similarity index 96% rename from dist/sampledata/Script.Example.scala rename to dist/sampledata/GeneralScripts/Script.Example.General.scala index 4b22a14..68c33ad 100644 --- a/dist/sampledata/Script.Example.scala +++ b/dist/sampledata/GeneralScripts/Script.Example.General.scala @@ -23,7 +23,7 @@ var taskDescription = """ """; var T = Task.fromString(taskDescription); -TaskRunner.runTask(T); +TaskRunner.runTaskAndWait(T); var allTasks = Task.allTasks(); diff --git a/dist/sampledata/QLObjects/Controller.Example.scala b/dist/sampledata/QLObjects/Controller.Example.scala new file mode 100644 index 0000000..4cd281a --- /dev/null +++ b/dist/sampledata/QLObjects/Controller.Example.scala @@ -0,0 +1,32 @@ + +// Determine Pricing Config for all trades +Steps ::= "Step_1" -> ((obj: SQObject) => { + + obj match { + case x:SwapTrade => { + println("Preparing trade " + x.ID) + determineConfig(x)} + + case _ => + } + +}) + +// Load Fixings, build curves and price the trades +Steps ::= "Step_2" -> ((obj: SQObject) => { + + obj match { + case x:Index => {println("Loading fixings for " + x.ID) + loadFixings(x) } + + case x:YieldCurve => {println("Building curve " + x.ID) + buildCurve(x) } + + case x:SwapTrade => {println("Building trade " + x.ID) + buildTrade(x) + println("Pricing trade " + x.ID) + QLSession.get[Instrument](x.ID).NPV()} + case _ => null + } + +}) \ No newline at end of file diff --git a/dist/sampledata/config.json b/dist/sampledata/QLObjects/Parametrization/Curves.json similarity index 96% rename from dist/sampledata/config.json rename to dist/sampledata/QLObjects/Parametrization/Curves.json index 3a926b7..7de8ecf 100644 --- a/dist/sampledata/config.json +++ b/dist/sampledata/QLObjects/Parametrization/Curves.json @@ -43,15 +43,6 @@ }], "TypeName":"YieldCurve" }, - "OISEurCollateral":{ - "jsonClass":"Schema$Mapping", - "ID":"OISEurCollateral", - "Pairs":{ - "DiscountCurve_EUR":"EUR_OIS", - "ForwardCurve_Euribor6M":"EUR_FWD_6M" - }, - "TypeName":"Mapping" - }, "EUR_6M_SWP_2Y":{ "jsonClass":"Schema$CollateralizedVanillaSwapInstrument", "ID":"EUR_6M_SWP_2Y", diff --git a/dist/sampledata/QLObjects/Parametrization/Mappings.json b/dist/sampledata/QLObjects/Parametrization/Mappings.json new file mode 100644 index 0000000..8eefee6 --- /dev/null +++ b/dist/sampledata/QLObjects/Parametrization/Mappings.json @@ -0,0 +1,11 @@ +{ + "OISEurCollateral":{ + "jsonClass":"Schema$Mapping", + "ID":"OISEurCollateral", + "Pairs":{ + "DiscountCurve_EUR":"EUR_OIS", + "ForwardCurve_Euribor6M":"EUR_FWD_6M" + }, + "TypeName":"Mapping" + } +} \ No newline at end of file diff --git a/dist/sampledata/QLObjects/Task.QuantlibObjects.json b/dist/sampledata/QLObjects/Task.QuantlibObjects.json new file mode 100644 index 0000000..ec80133 --- /dev/null +++ b/dist/sampledata/QLObjects/Task.QuantlibObjects.json @@ -0,0 +1,24 @@ +{ + "Interpreter" : "org.quil.interpreter.QuantLibObjects.QLObjectsInterpreter", + + "Task" : "RunQLObjectsApplication", + + "MarketData" : { + "Base" :"ExampleMarket", + "Additional" : { + "EUR_Swap_1Y" : "0.05" + } + }, + + "Trades" : { + "From" : "QLObjects/Trades/trades.json", + "Where" : " true " + }, + + "Parametrization" : [ + "QLObjects/Parametrization/Curves.json", + "QLObjects/Parametrization/Mappings.json" + ], + + "Controller" : "QLObjects/Controller.Example.scala" +} \ No newline at end of file diff --git a/dist/sampledata/trades.json b/dist/sampledata/QLObjects/Trades/trades.json similarity index 100% rename from dist/sampledata/trades.json rename to dist/sampledata/QLObjects/Trades/trades.json diff --git a/dist/sampledata/Script.Pricer.scala b/dist/sampledata/QLScriptPricer/Script.QuantlibScript.ExamplePricer.scala similarity index 91% rename from dist/sampledata/Script.Pricer.scala rename to dist/sampledata/QLScriptPricer/Script.QuantlibScript.ExamplePricer.scala index 1cf321e..f02d6b3 100644 --- a/dist/sampledata/Script.Pricer.scala +++ b/dist/sampledata/QLScriptPricer/Script.QuantlibScript.ExamplePricer.scala @@ -39,9 +39,12 @@ class Script extends QuantLibScript { val analyticEngine = new AnalyticHestonEngine(hestonModel) vanillaOption.setPricingEngine(analyticEngine) + println("Notional = " + tradeData.Notional.asString.toDouble) + + Document( Map( - "PV" -> (vanillaOption.NPV() * tradeData.Notional.asString.toDouble) + "PV" -> (vanillaOption.NPV() ) ) ) diff --git a/dist/sampledata/Task.PriceTrades.QLScript.json b/dist/sampledata/QLScriptPricer/Task.PriceTrades.QLScript.json similarity index 100% rename from dist/sampledata/Task.PriceTrades.QLScript.json rename to dist/sampledata/QLScriptPricer/Task.PriceTrades.QLScript.json diff --git a/dist/sampledata/Task.QuantLibScript.json b/dist/sampledata/QLScriptPricer/Task.QuantLibScript.json similarity index 100% rename from dist/sampledata/Task.QuantLibScript.json rename to dist/sampledata/QLScriptPricer/Task.QuantLibScript.json diff --git a/dist/sampledata/Task.PriceSingleTrade.json b/dist/sampledata/QLXMLTemplates/Task.PriceSingleTrade.json similarity index 100% rename from dist/sampledata/Task.PriceSingleTrade.json rename to dist/sampledata/QLXMLTemplates/Task.PriceSingleTrade.json diff --git a/dist/sampledata/Task.PriceTrades.json b/dist/sampledata/QLXMLTemplates/Task.PriceTrades.json similarity index 100% rename from dist/sampledata/Task.PriceTrades.json rename to dist/sampledata/QLXMLTemplates/Task.PriceTrades.json diff --git a/dist/sampledata/Template.Trade.Swap.xml b/dist/sampledata/QLXMLTemplates/Template.Trade.Swap.xml similarity index 100% rename from dist/sampledata/Template.Trade.Swap.xml rename to dist/sampledata/QLXMLTemplates/Template.Trade.Swap.xml diff --git a/dist/sampledata/Task.QuantlibObjects.json b/dist/sampledata/Task.QuantlibObjects.json deleted file mode 100644 index 4801c8c..0000000 --- a/dist/sampledata/Task.QuantlibObjects.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Interpreter" : "org.quil.interpreter.QuantLibObjects.QLObjectsInterpreter", - "Task" : "PriceTrade", - - "Parametrization" : { - Repository : "QLObjects", - Elements : ["Curves", "Mappings"] - }, - - "MarketData" : { - "Base" :"ExampleMarket", - "Additional" : { - "EUR_Swap_1Y" : "0.05" - } - }, - - "Trades" : "Trades", - - "Where" : " true " -} \ No newline at end of file diff --git a/interpreter/src/main/java/org/quil/interpreter/QuantLibScript/QuantLibScriptInterpreter.java b/interpreter/src/main/java/org/quil/interpreter/QuantLibScript/QuantLibScriptInterpreter.java index 9daf6f0..0b603e8 100644 --- a/interpreter/src/main/java/org/quil/interpreter/QuantLibScript/QuantLibScriptInterpreter.java +++ b/interpreter/src/main/java/org/quil/interpreter/QuantLibScript/QuantLibScriptInterpreter.java @@ -15,8 +15,6 @@ import org.springframework.context.support.FileSystemXmlApplicationContext; - - public class QuantLibScriptInterpreter implements Interpreter { private boolean _error = false; @@ -37,12 +35,15 @@ public QuantLibScriptInterpreter() { public QuantLibScript compile(scala.tools.nsc.Interpreter intp, String script, String ID, int hashCode) { - intp.compileString(script); + try { + + intp.compileString(script); Class compiledClass = intp.classLoader().loadClass("Script_"+ID); compiledClasses.put(hashCode, compiledClass); QuantLibScript scriptClass = (QuantLibScript)compiledClass.newInstance(); return scriptClass; + } catch (InstantiationException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -52,6 +53,9 @@ public QuantLibScript compile(scala.tools.nsc.Interpreter intp, String script, S } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); + }catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); }; return null; diff --git a/server/src/main/java/org/quil/server/ResultsCache.java b/interpreter/src/main/java/org/quil/server/ResultsCache.java similarity index 100% rename from server/src/main/java/org/quil/server/ResultsCache.java rename to interpreter/src/main/java/org/quil/server/ResultsCache.java diff --git a/server/src/main/java/org/quil/server/Tasks/PricePortfolio.java b/interpreter/src/main/java/org/quil/server/Tasks/PricePortfolio.java similarity index 100% rename from server/src/main/java/org/quil/server/Tasks/PricePortfolio.java rename to interpreter/src/main/java/org/quil/server/Tasks/PricePortfolio.java diff --git a/server/src/main/java/org/quil/server/Tasks/PriceTrade.java b/interpreter/src/main/java/org/quil/server/Tasks/PriceTrade.java similarity index 100% rename from server/src/main/java/org/quil/server/Tasks/PriceTrade.java rename to interpreter/src/main/java/org/quil/server/Tasks/PriceTrade.java diff --git a/server/src/main/java/org/quil/server/Tasks/ResultItem.java b/interpreter/src/main/java/org/quil/server/Tasks/ResultItem.java similarity index 100% rename from server/src/main/java/org/quil/server/Tasks/ResultItem.java rename to interpreter/src/main/java/org/quil/server/Tasks/ResultItem.java diff --git a/server/src/main/java/org/quil/server/Tasks/ScriptedTask.java b/interpreter/src/main/java/org/quil/server/Tasks/ScriptedTask.java similarity index 100% rename from server/src/main/java/org/quil/server/Tasks/ScriptedTask.java rename to interpreter/src/main/java/org/quil/server/Tasks/ScriptedTask.java diff --git a/server/src/main/java/org/quil/server/Tasks/Task.java b/interpreter/src/main/java/org/quil/server/Tasks/Task.java similarity index 98% rename from server/src/main/java/org/quil/server/Tasks/Task.java rename to interpreter/src/main/java/org/quil/server/Tasks/Task.java index 5409813..bb89b49 100644 --- a/server/src/main/java/org/quil/server/Tasks/Task.java +++ b/interpreter/src/main/java/org/quil/server/Tasks/Task.java @@ -1,13 +1,5 @@ package org.quil.server.Tasks; -import java.io.Serializable; -import java.lang.reflect.Constructor; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import javax.cache.Cache.Entry; - import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteLogger; @@ -19,8 +11,12 @@ import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import javax.cache.Cache.Entry; +import java.io.Serializable; +import java.lang.reflect.Constructor; +import java.util.HashMap; +import java.util.UUID; public abstract class Task implements Serializable { diff --git a/qlobjects/src/main/main.iml b/qlobjects/src/main/main.iml new file mode 100644 index 0000000..19dbd15 --- /dev/null +++ b/qlobjects/src/main/main.iml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/qlobjects/src/main/scala/org/quil/interpreter/QuantLibObjects/Controller.scala b/qlobjects/src/main/scala/org/quil/interpreter/QuantLibObjects/Controller.scala new file mode 100644 index 0000000..3af7c71 --- /dev/null +++ b/qlobjects/src/main/scala/org/quil/interpreter/QuantLibObjects/Controller.scala @@ -0,0 +1,8 @@ +package org.quil.interpreter.QuantLibObjects + +/** + * Created by d90590 on 07.06.2016. + */ +class Controller { + +} diff --git a/qlobjects/src/main/scala/org/quil/interpreter/QuantLibObjects/QLObjectsInterpreter.scala b/qlobjects/src/main/scala/org/quil/interpreter/QuantLibObjects/QLObjectsInterpreter.scala index c1c41e5..e27dcbc 100644 --- a/qlobjects/src/main/scala/org/quil/interpreter/QuantLibObjects/QLObjectsInterpreter.scala +++ b/qlobjects/src/main/scala/org/quil/interpreter/QuantLibObjects/QLObjectsInterpreter.scala @@ -1,22 +1,153 @@ package org.quil.interpreter.QuantLibObjects +import java.io.{ByteArrayOutputStream, PrintStream, PrintWriter, StringWriter} + +import org.apache.ignite.Ignition import org.quil.interpreter._ +import org.json.simple._ +import org.json.simple.parser.JSONParser +import org.quil.repository.CachedFileSystemRepository +import org.quil.objects._ +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import scala.tools.nsc.Settings +import scala.tools.nsc.interpreter.IMain +import scala.tools.nsc.interpreter.Results.Result +import org.json4s.native.Serialization.write +import org.quil.interpreter.QuantLibTemplates.Market -import org.json.simple.JSONObject /** + * * Created by d90590 on 06.06.2016. */ class QLObjectsInterpreter extends org.quil.interpreter.Interpreter { + val logger = LoggerFactory.getLogger(classOf[QLObjectsInterpreter]) + var _data:JSONObject = new JSONObject() + var _result:JSONObject = new JSONObject() + var _error = false def setData(data:JSONObject) = { _data = data } - def getError(): Boolean = ??? - def getResult(): org.json.simple.JSONObject = ??? - def interpret(): Unit = ??? + def getError(): Boolean = _error + + def getResult(): org.json.simple.JSONObject = _result + + def interpret(): Unit = { + + val ignite = Ignition.ignite() + + try { + + logger.info("Loading Controller") + val controllerFileName = _data.get("Controller").asInstanceOf[String] + val controllerScript = CachedFileSystemRepository.instance().getFile(controllerFileName) + + val paramObjects = _data.get("Parametrization").asInstanceOf[JSONArray] + + import Schema._ + + logger.info("Loading Parametrization") + (0 to paramObjects.size()-1).foreach { i=> + loadFromString(CachedFileSystemRepository.instance().getFile(paramObjects.get(i).asInstanceOf[String])) + } + + logger.info("Loading Trades") + val trades = _data.get("Trades").asInstanceOf[JSONObject] + val from = trades.get("From").asInstanceOf[String] + + var idsLoaded = List[String]() + if (from.contains("/")) { + idsLoaded = loadFromString(CachedFileSystemRepository.instance().getFile(from)) + //TODO WHERE predicate + } else { + //TODO trades from other cache + //TODO WHERE predicate + } + + // Add trades to initial object + object initObject extends SQObject("InitObject") { + val ID = "InitObject" + val trades = idsLoaded.map( x=> SQReference(x, "SwapTrade")); + } + + //Load MD + val MD: Market = new Market + val marketData: JSONObject = _data.get("MarketData").asInstanceOf[JSONObject] + if (marketData != null) { + logger.info("Injecting market parameters.") + val base: String = marketData.get("Base").asInstanceOf[String] + MD.setBase(base) + val overrideMarketData: JSONObject = marketData.get("Additional").asInstanceOf[JSONObject] + if (overrideMarketData != null) { + val iterator = overrideMarketData.keySet.iterator + while (iterator.hasNext) { + { + val key: String = iterator.next.asInstanceOf[String] + MD.set(key, overrideMarketData.get(key).asInstanceOf[String]) + logger.info("Market delta: " + key + " = " + overrideMarketData.get(key).asInstanceOf[String]) + } + } + } + } + //load Steps + val script: String = _data.get("Script").asInstanceOf[String] + val init: String = """import scala.collection.JavaConversions._; + import org.quil.objects._; + import org.quil.objects.Schema._; + import org.quantlib.{Array=>QArray, Index=>QIndex, _}; """ + + Thread.currentThread.setContextClassLoader(this.getClass.getClassLoader) + val settings: Settings = new Settings + settings.usejavacp.tryToSetFromPropertyValue("true") + settings.embeddedDefaults(Thread.currentThread().getContextClassLoader) + val baos: ByteArrayOutputStream = new ByteArrayOutputStream + val ps: PrintStream = new PrintStream(baos) + val out: StringWriter = new StringWriter + val stream: PrintWriter = new PrintWriter(out) + val imain: IMain = new IMain(settings, stream) + + imain.interpret(init) + + var r: Result = imain.bind("MD", "org.quil.interpreter.QuantLibTemplates.Market", MD) + println(r) + + var oldOut = Console.out + imain.eval("Console").asInstanceOf[Console.type].setOut(baos) + imain.interpret("var Steps = List[(String,(SQObject=>Any))]()\n" + controllerScript) + + val steps = imain.eval("Steps").asInstanceOf[List[(String,(org.quil.objects.SQObject=>Any))]] + + var res = List[(String,Any)]() + steps.reverse.foreach {step => + logger.info("Executing Step '"+step._1+"'") + res :::= Graph.applyOnce(initObject, step._2) + } + + Console.setOut(oldOut) + + + implicit val formats = serializationFormat() + val resStr = s"""{ "EvaluationResult" : ${write(res)}, + "Output" : ${write(baos.toString)}, + "ReplOutput" : ${write(out.toString)} + }""" + + println (resStr); + + _result = new JSONParser().parse(resStr).asInstanceOf[JSONObject] + + } catch { + case e:Exception => { + _error = true + throw e } + } + + } } diff --git a/qlobjects/src/main/scala/org/quil/server/Tasks/RunQLObjectsApplication.scala b/qlobjects/src/main/scala/org/quil/server/Tasks/RunQLObjectsApplication.scala new file mode 100644 index 0000000..66def12 --- /dev/null +++ b/qlobjects/src/main/scala/org/quil/server/Tasks/RunQLObjectsApplication.scala @@ -0,0 +1,54 @@ +package org.quil.server.Tasks + +import org.apache.ignite.cache.CacheMode +import org.apache.ignite.{Ignite, Ignition} +import org.apache.ignite.configuration.CacheConfiguration +import org.json.simple.JSONObject +import org.json.simple.parser.JSONParser +import org.quil.interpreter.Interpreter +import org.quil.server.ResultsCache + +/** + * Created by d90590 on 07.06.2016. + */ + +object RunQLObjectsApplication { + +} + +class RunQLObjectsApplication(val taskName:String, val taskDescription:String) extends + org.quil.server.Tasks.Task(taskName,taskDescription) { + + + def run() = { + + val parser: JSONParser = new JSONParser + + val taskDescription: JSONObject = parser.parse(_taskDescription).asInstanceOf[JSONObject] + + val interpreter: Interpreter = Class.forName(taskDescription.get("Interpreter").asInstanceOf[String]).newInstance.asInstanceOf[Interpreter] + interpreter.setData(taskDescription) + interpreter.interpret + + Task.updateResult(_taskName, interpreter.getResult.toJSONString) + import scala.collection.JavaConversions._ + for (r <- interpreter.getResult.keySet) { + val key: String = r.asInstanceOf[String] + var doubleVal: Double = 0.0 + var intVal: Int = 0 + var strVal: String = "" + try { + doubleVal = interpreter.getResult.get(key).asInstanceOf[String].toDouble + intVal = interpreter.getResult.get(key).asInstanceOf[String].toInt + strVal = interpreter.getResult.get(key).asInstanceOf[String] + } + catch { + case e: Exception => { + } + } + ResultsCache.add(_taskName, _taskTag, 0, key, strVal, doubleVal, intVal) + } + + if (interpreter.getError) throw new Exception("Error during interpretation in task PriceTrade.") + } +} diff --git a/server/src/main/java/org/quil/server/ClusterAPI.java b/server/src/main/java/org/quil/server/ClusterAPI.java index 382451f..9369c09 100644 --- a/server/src/main/java/org/quil/server/ClusterAPI.java +++ b/server/src/main/java/org/quil/server/ClusterAPI.java @@ -4,11 +4,8 @@ import java.util.HashMap; import java.util.Map; -import javax.ws.rs.Consumes; import javax.ws.rs.GET; -import javax.ws.rs.POST; import javax.ws.rs.Path; -import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; @@ -18,8 +15,6 @@ import org.apache.ignite.cluster.ClusterNode; import org.json.simple.JSONArray; import org.json.simple.JSONObject; -import org.quil.server.Tasks.Task; -import org.quil.server.Tasks.TaskRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/server/src/main/java/org/quil/server/QuilServer.java b/server/src/main/java/org/quil/server/QuilServer.java index a773eb2..6f16976 100644 --- a/server/src/main/java/org/quil/server/QuilServer.java +++ b/server/src/main/java/org/quil/server/QuilServer.java @@ -11,6 +11,7 @@ import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.webapp.WebAppContext; +import org.quil.repository.CachedFileSystemRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.ignite.Ignition; @@ -59,7 +60,7 @@ public static void main(String[] args) throws Exception { try { // TODO Make configurable - TcpDiscoverySpi spi = new TcpDiscoverySpi(); + /*TcpDiscoverySpi spi = new TcpDiscoverySpi(); TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(); ipFinder.setAddresses(Arrays.asList("127.0.0.1", "127.0.0.1:47500..47509")); spi.setIpFinder(ipFinder); @@ -69,7 +70,8 @@ public static void main(String[] args) throws Exception { FifoQueueCollisionSpi spiCollision = new FifoQueueCollisionSpi(); spiCollision.setParallelJobsNumber(4); cfg.setCollisionSpi(spiCollision); - + + */ if (!workerNode) { boolean clientmode = true; try { @@ -84,15 +86,25 @@ public static void main(String[] args) throws Exception { logger.info("Client mode = " + clientmode); - cfg.setClientMode(clientmode); + if (clientmode) + Ignition.start("config/quil-client.xml"); + else + Ignition.start("config/quil-server.xml"); + + //cfg.setClientMode(clientmode); } + else { + Ignition.start("config/quil-server.xml"); + } - cfg.setPeerClassLoadingEnabled(true); - cfg.setIncludeEventTypes(EVTS_TASK_EXECUTION); - - Ignition.start(cfg); + //cfg.setPeerClassLoadingEnabled(true); + //cfg.setIncludeEventTypes(EVTS_TASK_EXECUTION);*/ + + runQuilStartupScript(); + + CachedFileSystemRepository.instance(); if (!workerNode) { jettyServer.start(); diff --git a/server/src/main/java/org/quil/server/RepositoryAPI.java b/server/src/main/java/org/quil/server/RepositoryAPI.java index 66510a0..a2f7f8b 100644 --- a/server/src/main/java/org/quil/server/RepositoryAPI.java +++ b/server/src/main/java/org/quil/server/RepositoryAPI.java @@ -1,10 +1,5 @@ package org.quil.server; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; @@ -12,15 +7,8 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; -import org.apache.ignite.Ignite; -import org.apache.ignite.Ignition; -import org.apache.ignite.cluster.ClusterMetrics; -import org.apache.ignite.cluster.ClusterNode; -import org.json.simple.JSONArray; import org.json.simple.JSONObject; -import org.quil.repository.FileSystemRepository; -import org.quil.server.Tasks.Task; -import org.quil.server.Tasks.TaskRunner; +import org.quil.repository.CachedFileSystemRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +25,7 @@ public String content() { { // TODO Re-Desgn Repo Class (still ugly) JSONObject repo = new JSONObject(); - (new FileSystemRepository()).genRepositoryObject(null, repo, ""); + CachedFileSystemRepository.instance().genRepositoryObject(null, repo, ""); return repo.toJSONString(); } catch (Exception e) @@ -53,7 +41,7 @@ public String getFile(@PathParam("path") String path) { try { JSONObject fileData = new JSONObject(); - fileData.put("fileData", (new FileSystemRepository()).getFile(path)); + fileData.put("fileData", CachedFileSystemRepository.instance().getFile(path)); return fileData.toJSONString(); } catch (Exception e) @@ -68,7 +56,7 @@ public String getFile(@PathParam("path") String path) { public String putFile(@PathParam("path") String path, String content) { try { - (new FileSystemRepository()).putFile(path, content); + CachedFileSystemRepository.instance().putFile(path, content); return success(); } @@ -85,7 +73,7 @@ public String putFile(@PathParam("path") String path, String content) { public String deleteFile(@PathParam("path") String path) { try { - (new FileSystemRepository()).deleteFile(path); + CachedFileSystemRepository.instance().deleteFile(path); return success(); } @@ -102,7 +90,7 @@ public String deleteFile(@PathParam("path") String path) { public String createFolder(@PathParam("path") String path) { try { - (new FileSystemRepository()).putFolder(path); + CachedFileSystemRepository.instance().putFolder(path); return success(); } @@ -119,7 +107,7 @@ public String createFolder(@PathParam("path") String path) { public String deleteFolder(@PathParam("path") String path) { try { - (new FileSystemRepository()).deleteFile(path); + CachedFileSystemRepository.instance().deleteFile(path); return success(); } diff --git a/server/src/main/java/org/quil/server/TaskAPI.java b/server/src/main/java/org/quil/server/TaskAPI.java index e09873f..032f08e 100644 --- a/server/src/main/java/org/quil/server/TaskAPI.java +++ b/server/src/main/java/org/quil/server/TaskAPI.java @@ -13,8 +13,8 @@ import org.json.simple.JSONArray; import org.json.simple.JSONObject; -import org.quil.server.Tasks.Task; import org.quil.server.Tasks.TaskRunner; +import org.quil.server.Tasks.Task; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/webapp/src/main/webapp/js/controllers.js b/webapp/src/main/webapp/js/controllers.js index d251c48..daea232 100644 --- a/webapp/src/main/webapp/js/controllers.js +++ b/webapp/src/main/webapp/js/controllers.js @@ -734,11 +734,20 @@ controllers.controller("TaskActionController", ['$scope', '$state', '$http', fu $scope.loadDefinedTasks = function() { $http.get('/api/repository/content'). success(function(data, status, headers, config) { - for (var i=0; i < data.children.length; i++) { - - if (data.children[i].id.indexOf("Task.") != -1 || data.children[i].id.indexOf(".scala") != -1) - $scope.definedTasks.push(data.children[i]); - } + + var findTasks = function(node) { + + for (var i=0; i < node.children.length; i++) { + if (node.children[i].id.indexOf("Task.") != -1 || node.children[i].id.indexOf(".scala") != -1) + $scope.definedTasks.push(node.children[i]); + + if (node.children[i].icon.indexOf("folder")!=-1) + findTasks(node.children[i]) + } + }; + + findTasks(data); + }). error(function(data, status, headers, config) { console.log("Failed to get repo!");