diff --git a/CHANGES.txt b/CHANGES.txt index 39c3104e..79cf1734 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,6 +5,16 @@ Changes for Crate Admin Interface Unreleased ========== +2018/08/27 1.10.3 +================= + +Feature +------- + +- sharding-calculator: a simple tool to walk a user through the process, how + to split their crate table into shards. Can also read stats from existing table. + + 2018/06/29 1.10.3 ================= diff --git a/app/conf/plugins.json b/app/conf/plugins.json old mode 100644 new mode 100755 index f53e5a6a..279dcc64 --- a/app/conf/plugins.json +++ b/app/conf/plugins.json @@ -53,4 +53,17 @@ "controller": "ShardsController" } } +}, { + "name": "calculator", + "uri": "static/plugins/tutorial/calculator.js", + "stylesheet": "static/plugins/tutorial/calculator.css", + "enterprise": false, + "routing": { + "/calculator": { + "name": "calculator", + "url": "/calculator", + "templateUrl": "static/plugins/tutorial/calculator.html", + "controller": "CalculatorController" + } + } }] diff --git a/app/plugins/tutorial/calculator.css b/app/plugins/tutorial/calculator.css new file mode 100644 index 00000000..9a5b9f64 --- /dev/null +++ b/app/plugins/tutorial/calculator.css @@ -0,0 +1,103 @@ + +.cr-panel-block--calculator{ /* refer to .cr-panel-block--info */ + /* big box */ + background-color: #353535; + margin-top: 10px; + margin-bottom: 10px; + padding-top: 0px; + padding-right: 10px; + padding-bottom: 10px; + padding-left: 10px; + border: 1px solid #545454; +} + +.cr-panel-block__calculator{ /* refer to cr-panel-block__item */ + margin: 5px; +} +.cr-panel-block__calculator__header{ + padding-bottom: 3px; + letter-spacing: 0.6px; + font-weight: bold; + background-color: #353535; + font-size: 10px; +} + +.cr-panel-block__calculator__content{ + /* small box */ + width:100%; + margin-left:10px; + margin-right:10px; + margin-top:0px; + margin-bottom:0px; + padding: 7px; + font-size: 13px; + overflow: hidden; + word-break: normal; + border: 1px solid #242424 !important; + border-radius: 3px; + background: #242424 !important; +} + + +.result { + font-size:14px; + font-weight: bold; + color: #55d4f5; /*crate blue*/ +} + + +.last_row { + padding-bottom:0px !important; + margin-bottom:0px !important; +} + +.column { + float: left; + width: 50%; + padding: 5px; +} + +/* Clear floats after the columns */ +.row:after { + content: ""; + display: table; + clear: both; +} + +input{ + width:50px; + background-color:#545454; + border: 1px solid #545454; + border-radius: 4px; +} + + +input:disabled { + color: #606060; +} + +select { + width:auto; + background-color:#545454; + border: 1px solid #545454; + border-radius: 4px; +} + +select:disabled { + color: #606060; +} + +.crb-conatiner { /*crb ... cool-radio-button*/ + width: 100%; + overflow: hidden; +} + +.crb-button { /*crb ... cool-radio-button*/ + float: left; + padding-left: -25px; + margin-right: -15px; +} + +.crb-description { /*crb ... cool-radio-button*/ + padding-top: 0.3%; +} \ No newline at end of file diff --git a/app/plugins/tutorial/calculator.html b/app/plugins/tutorial/calculator.html new file mode 100755 index 00000000..94df472e --- /dev/null +++ b/app/plugins/tutorial/calculator.html @@ -0,0 +1,319 @@ +
+
+

+ {{:: 'CALCULATOR.TITLE' | translate}} +

+

+ {{:: 'CALCULATOR.DESCRIPTION' | translate}} +

+ +
+
+
+

{{:: 'CALCULATOR.IMPORT_HEADING' | translate}}

+ +
+ +
+

+ {{:: 'CALCULATOR.HARDWARE_HEADING' | translate}} +

+ + +
+

+ {{:: 'CALCULATOR.CPUS_HEADING' | translate}} +

+

+ {{:: 'CALCULATOR.CPUS_DESCRIPTION' | translate}} +

+
+ +
+
+ + +
+

+ {{:: 'CALCULATOR.STORAGE_HEADING' | translate}} +

+
+ {{:: 'CALCULATOR.STORAGE_DESCRIPTION' | translate}} +
+
+ + +
+

+ {{:: 'CALCULATOR.RAMSTORAGE_HEADING' | translate}} +

+

+ {{:: 'CALCULATOR.RAMSTORAGE_DESCRIPTION' | translate}} +

+ +
+ + {{:: 'CALCULATOR.RAMSTORAGE_INPUT_1' | translate}} + + + + {{:: 'CALCULATOR.RAMSTORAGE_INPUT_2' | translate}} + +
+
+ + +
+

+ {{:: 'CALCULATOR.RAM_HEADING' | translate}} +

+
+

+ {{:: 'CALCULATOR.RAM_DESCRIPTION_1' | translate}} +

+
+ +

+ + + + + + {{:: 'CALCULATOR.RAM_DESCRIPTION_2' | translate}} + +

+ +

+ {{:: 'CALCULATOR.RAM_DESCRIPTION_3' | translate}} +

+ +
+
+
+ + +
+

+ {{:: 'CALCULATOR.RAID_HEADING' | translate}} +

+
+ {{:: 'CALCULATOR.RAID_DESCRIPTION' | translate}} +
+
+
+
+ + + + + +
+
+

+ {{:: 'CALCULATOR.USECASE_HEADING' | translate}} +

+ + +
+

+ {{:: 'CALCULATOR.DATA_HEADING' | translate}} +

+

+ {{:: 'CALCULATOR.DATA_DESCRIPTION_1' | translate}} +

+ +
+

+

+
+ +
+
+ {{:: 'CALCULATOR.DATA_DESCRIPTION_2' | translate}} +
+
+

+ +
+ + + + + + + {{:: 'CALCULATOR.DATA_DESCRIPTION_3' | translate}} + + + +

+

+ {{:: 'CALCULATOR.DATA_DESCRIPTION_4' | translate}} +

+

+ + + +

+
+
+ +

+ {{:: 'CALCULATOR.DATA_DESCRIPTION_5' | translate}} +

+ +
+

+

+
+ +
+
+ {{:: 'CALCULATOR.DATA_DESCRIPTION_6' | translate}} +
+
+

+ +
+ + + + + + + {{:: 'CALCULATOR.DATA_DESCRIPTION_7' | translate}} + + +
+
+
+ + + +
+

+ {{:: 'CALCULATOR.PARTITION_HEADING' | translate}} +

+

+ {{:: 'CALCULATOR.PARTITION_DESCRIPTION_1' | translate}} +

+ +
+
+

+ {{:: 'CALCULATOR.PARTITION_DESCRIPTION_2' | translate}}

+ + + + +
+ +
+

+ {{:: 'CALCULATOR.PARTITION_DESCRIPTION_3' | translate}}

+ + partitions. + +
+
+
+ + + + +
+

+ {{:: 'CALCULATOR.REDUNDANCY_HEADING' | translate}} +

+

+ {{:: 'CALCULATOR.REDUNDANCY_DESCRIPTION' | translate}} +

+ +
+ +
+
+
+
+
+ + + +
+

+ {{:: 'CALCULATOR.CALCULATION_HEADING' | translate}} +

+

+ {{:: 'CALCULATOR.CALCULATION_DESCRIPTION' | translate}} +

+ + +
+

+ {{:: 'CALCULATOR.NODES_HEADING' | translate}} +

+
+ {{:: 'CALCULATOR.NODES_DESCRIPTION_1' | translate}} + + {{:: 'CALCULATOR.NODES_DESCRIPTION_2' | translate}} + + {{selectedRAM() | bytes:0}} + + {{:: 'CALCULATOR.NODES_DESCRIPTION_3' | translate}} + + {{storagePerNode() | bytes:2}} + + {{:: 'CALCULATOR.NODES_DESCRIPTION_4' | translate}} +
+
+ + + +
+

+ {{:: 'CALCULATOR.SHARDS_HEADING' | translate}} +

+
+ {{:: 'CALCULATOR.SHARDS_DESCRIPTION_1' | translate}} + + {{:: 'CALCULATOR.SHARDS_DESCRIPTION_2' | translate}} +
+
+
+
+
\ No newline at end of file diff --git a/app/plugins/tutorial/calculator.js b/app/plugins/tutorial/calculator.js new file mode 100755 index 00000000..2dc83b82 --- /dev/null +++ b/app/plugins/tutorial/calculator.js @@ -0,0 +1,258 @@ +'use strict'; + +// storage is measured in Bytes +// time is measured in hours + +angular.module('calculator', ['sql', 'translation']).controller('CalculatorController', function($scope, SQLQuery, queryResultToObjects) { + var diskLoadFactor = 0.85; + var maxRAMPerNode = 64000000000; //64G + $scope.RAMInput = 64; + $scope.RAMInputUnitPrefix = 'Gibi'; + $scope.hideGCHint = true; + var sizeFactor = 0.732; //from haudi's document + var maxShardSize = 32000000000; //32G, compromise from haudi's and andrei's opinions + var maxShards = 1000; + $scope.CPUCoresPerNode = 2; + $scope.RAMStorageProportion = 24; + $scope.dataType = 'perTime'; + $scope.dataInsertedPerTime = 20; + $scope.expectedTableSize = 2; + $scope.expectedTableSizeUnitPrefix = 'Tebi'; + $scope.dataInsertedPerTimeUnitPrefix = 'Gibi'; + $scope.dataInsertedPerTimeTemporalUnit = 'day'; + $scope.keepTimeTemporalUnit = 'month'; + $scope.keepTime = 6; + $scope.partitionSize = 1; + $scope.partitionSizeTemporalUnit = 'month'; + $scope.manualPartitionCount = 4; + $scope.replicas = 1; + $scope.tables = []; + var selectSchema = "none"; + var selectTable = "none"; + $scope.selected = "none"; + + $scope.selectedRAM = function (){ + var r = $scope.RAMInput * prefix($scope.RAMInputUnitPrefix); + $scope.hideGCHint = r <= maxRAMPerNode; + return r; + }; + var neededDiskSpace = function () { + var res = 1; + if ($scope.dataType === 'absolute') { + res = ($scope.expectedTableSize * prefix($scope.expectedTableSizeUnitPrefix) * (1 + Number($scope.replicas))) / sizeFactor / diskLoadFactor; + } else if ($scope.dataType === 'perTime') { + res = ((prefix($scope.dataInsertedPerTimeUnitPrefix) * $scope.dataInsertedPerTime / temporalUnit($scope.dataInsertedPerTimeTemporalUnit)) * $scope.keepTime * temporalUnit($scope.keepTimeTemporalUnit) * (1 + Number($scope.replicas))) / diskLoadFactor / sizeFactor; //explicit cast of replica to number is necessary, otherwise 1+1=11. thanks java script + } + return res; + }; + $scope.neededNodes = function () { + return Math.ceil((neededDiskSpace() / $scope.RAMStorageProportion) / $scope.selectedRAM()); + }; + $scope.partitions = function () { + var res = 1; + if ($scope.dataType === 'perTime') { + res = (($scope.keepTime * temporalUnit($scope.keepTimeTemporalUnit)) / ($scope.partitionSize * temporalUnit($scope.partitionSizeTemporalUnit))); + } else if ($scope.dataType === 'absolute') { + res = $scope.manualPartitionCount; + } + return res; + }; + var shards = function () { + return Math.ceil(($scope.neededNodes() * $scope.CPUCoresPerNode) / $scope.partitions()); + }; + var shardSize = function (shards) { + return neededDiskSpace() / (shards * $scope.partitions() * (1 + Number($scope.replicas))); + }; + $scope.storagePerNode = function () { + if ($scope.neededNodes()!==0){ + return Number(neededDiskSpace()) / Number($scope.neededNodes()); + } + else{ + return 0; + } + }; + var prefix = function (x) { + switch (x) { + case "Tebi": + return Math.pow(2, 40); + case "Gibi": + return Math.pow(2, 30); + case "Mebi": + return Math.pow(2, 20); + case "Kibi": + return Math.pow(2, 10); + default: + return Math.pow(10, 0); + } + }; + var temporalUnit = function (x) { + switch (x) { + case "hour": + return 1; + case "day": + return 24; + case "week": + return 7 * 24; + case "month": + return 30 * 24; + case "year": + return 365 * 24; + default: + return 1; + } + }; + $scope.result = function () { + var s = shards(); + if (shardSize(s) > maxShardSize) { + s = Math.ceil(s * (shardSize(s) / maxShardSize)); + } + if (s > maxShards) { + return "maximum shard limit exceeded, please talk to an crate engineer about your use-case"; + } + return s; + }; + + + $scope.gettablename = function() { + var stmt = "SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema NOT IN ('information_schema', 'pg_catalog', 'sys', 'blob') order by table_schema, table_name"; + var cols = ['table_name', 'schema_name']; + var obj = []; + SQLQuery.execute(stmt, {}, false, false, false, false).then(function (query) { + $scope.sqlresult = queryResultToObjects(query, cols); + for(var i=0; i<$scope.sqlresult.length; i++) { + obj.push([$scope.sqlresult[i].schema_name, $scope.sqlresult[i].table_name]); + } + $scope.tables = obj; + }); + + }; + + $scope.tableSelected = function () { + selectSchema = $scope.selected[0]; + selectTable = $scope.selected[1]; + loadCPUCores(selectSchema, selectTable); + loadTablesize(selectSchema, selectTable); + loadPartition(selectSchema, selectTable); + loadReplica(selectSchema, selectTable); + loadRAMStoragePropotion(); + loadRAM(); + }; + var loadCPUCores = function (schemaName, tableName) { + var stmt = "SELECT os_info['available_processors']\n" + + "FROM sys.nodes limit 100;"; + SQLQuery.execute(stmt, {}, false, false, false, false).then(function (query) { + $scope.CPUCoresPerNode = (query.rows[0])[0]; //we get a 2d array returned + }); + }; + + var loadTablesize = function(schemaName, tableName) { + var stmt = "select sum(size) from sys.shards where schema_name = '" + schemaName + + "'and table_name = '"+tableName+"' and primary=true;"; + SQLQuery.execute(stmt, {}, false, false, false, false).then(function (query) { + if ((query.rows[0])[0]===null){ + $scope.expectedTableSize = 0; + $scope.expectedTableSizeUnitPrefix = '1'; + $scope.dataType = 'absolute'; + return; + } + var size = (query.rows[0])[0]; + $scope.expectedTableSize = Number(getPrefixedNumber(size)); + $scope.expectedTableSizeUnitPrefix = getPrefix(size); + $scope.dataType = 'absolute'; + }); + }; + + var loadPartition = function(schemaName, tableName) { + var stmt = "select partitioned_by from information_schema.tables where table_schema = '" + +schemaName+"' and table_name = '"+tableName+"';"; + SQLQuery.execute(stmt, {}, false, false, false, false).then(function (query) { + if((query.rows[0])[0]!==null){ + stmt = "SELECT COUNT(*) FROM(select * from information_schema.table_partitions WHERE schema_name = '" + +schemaName+"' and table_name = '"+tableName+"') as x;"; + SQLQuery.execute(stmt, {}, false, false, false, false).then(function (query) { + $scope.manualPartitionCount = (query.rows[0])[0]; + }); + } + else{ + $scope.manualPartitionCount = 1; + } + }); + }; + + var loadReplica = function(schemaName, tableName) { + var rep = ""; + var stmt = "SELECT number_of_replicas FROM information_schema.tables WHERE table_schema='" + +schemaName+"' and table_name = '"+tableName+"';"; + SQLQuery.execute(stmt, {}, false, false, false, false).then(function (query) { + rep = (query.rows[0])[0]; + if(rep.includes("-") === true){ + rep = rep.split("-")[1]; + stmt = "SELECT COUNT(*) FROM sys.nodes"; + SQLQuery.execute(stmt, {}, false, false, false, false).then(function (query) { + if (rep ==="all"){ + $scope.replicas = (query.rows[0])[0]-1; + } + else{ + $scope.replicas = Math.min(Number(rep),(query.rows[0])[0]-1); + } + }); + } + else{ + $scope.replicas = Number(rep); + } + }); + }; + + var loadRAMStoragePropotion = function() { + var stmt = "SELECT fs['total']['available']/(heap['used']+heap['free']) AS RAMStoragePropotion FROM sys.nodes;"; + SQLQuery.execute(stmt, {}, false, false, false, false).then(function (query) { + var sum = 0; + for(var i = 0; i < query.rows.length; i++){ + sum += query.rows[i]; + } + var avg = Math.round(sum / query.rows.length); + $scope.RAMStorageProportion = avg; + }); + }; + + var getPrefix = function(x){ + if (x < Math.pow(2, 10)) { + return "1"; + } else if (x < Math.pow(2, 20)){ + return "Kibi"; + } else if (x < Math.pow(2, 30)){ + return "Mebi"; + } else if (x < Math.pow(2, 40)) { + return "Gibi"; + } else { + return "Tebi"; + } + }; + + var getPrefixedNumber = function(x){ + if (x < Math.pow(2, 10)) { + return x; + } else if (x < Math.pow(2, 20)){ + return (x / Math.pow(2, 10)).toFixed(1); + } else if (x < Math.pow(2, 30)){ + return (x / Math.pow(2, 20)).toFixed(1); + } else if (x < Math.pow(2, 40)) { + return (x / Math.pow(2, 30)).toFixed(1); + } else { + return (x / Math.pow(2, 40)).toFixed(1); + } + }; + + var loadRAM = function () { + var stmt = "SELECT (heap['used']+heap['free']) AS total_ram FROM sys.nodes;"; + SQLQuery.execute(stmt, {}, false, false, false, false).then(function (query) { + var sum = 0; + for(var i = 0; i < query.rows.length; i++){ + sum += Number(query.rows[i]); //thanks for adding objects here javascript + } + var avg = Math.round(sum / query.rows.length); + $scope.RAMInputUnitPrefix = getPrefix(avg); + $scope.RAMInput = Number(getPrefixedNumber(avg)); + }); + }; +}); diff --git a/app/plugins/tutorial/static/i18n/de.json b/app/plugins/tutorial/static/i18n/de.json index 311d6a7d..850179cc 100644 --- a/app/plugins/tutorial/static/i18n/de.json +++ b/app/plugins/tutorial/static/i18n/de.json @@ -4,11 +4,11 @@ }, "TUTORIAL": { "TITLE": "Mit Beispieldaten loslegen", - "PARAGRAPH_1": "Die Arbeit mit Beispieldaten hilft dir CrateDB etwas besser zu verstehen. Mit diesem Twitter-Importer ist das ganz einfach und du kannst schnell und mühelos deine neue CrateDB Instanz mit realen Daten testen. Klicke auf den Button, und beobachte wie dein Cluster in Echtzeit mit Live-Tweets gefüllt wird. Keine Sorge, diese können später immer noch gelöscht werden.", + "PARAGRAPH_1": "Die Arbeit mit Beispieldaten hilft dir, CrateDB etwas besser zu verstehen. Mit diesem Twitter-Importer ist das ganz einfach und du kannst schnell und mühelos deine neue CrateDB Instanz mit realen Daten testen. Klicke auf den Button, und beobachte wie dein Cluster in Echtzeit mit Live-Tweets gefüllt wird. Keine Sorge, diese können später immer noch gelöscht werden.", "PARAGRAPH_2": "Nach der Authentifizierung werden Tweets einzeln importiert, was natürlich langsam ist. Allerdings können in einer Produktionsumgebung bis zu mehreren tausend Datensätze pro Sekunde indiziert werden.", "INSTRUCTION_1": "Deine Tweets werden in der Tabelle {tweets} gespeichert", "TWEETS": "tweets", - "INSTRUCTION_2": "Klicke in der linken Menüleiste auf {tables} um alle Tabellen und deren Statistiken zu sehen. Du müsstest sehen, wie deren Datensätze steigen während Twitter importiert.", + "INSTRUCTION_2": "Klicke in der linken Menüleiste auf {tables} um alle Tabellen und deren Statistiken zu sehen. Du siehst, wie die Anzahl der Datensätze steigt während neue Daten von Twitter importiert werden.", "TABLES": "TABLES", "INSTRUCTION_3": "Klicke in der linken Menüleiste auf {console} um eine SQL-Abfrage zu starten. Zum Beispiel: {query}", "CONSOLE": "KONSOLE", @@ -16,5 +16,55 @@ "IMPORTING_TWEETS": "Importiere Tweets.", "TWEETS_IMPORTED": "Tweets importiert.", "STOP_IMPORTING_TWEETS": "Genug! Import gestoppt." + }, + "CALCULATOR": { + "TITLE": "Shard Rechner", + "DESCRIPTION": "Ein einfaches Werkzeug um eine generische Empfehlung für die Shard Größe für eine Tabelle in der CrateDB.", + "IMPORT_HEADING": "Daten von einer bereits existierenden Tabelle einlesen", + "HARDWARE_HEADING": "Hardware", + "CPUS_HEADING": "CPUs", + "CPUS_DESCRIPTION": "Wie viele CPU Kerne haben die Rechner auf welchen die Knoten laufen?", + "STORAGE_HEADING": "Präsistenter Speicher", + "STORAGE_DESCRIPTION": "Du verwendest SSDs anstadt HDDs? Gut.", + "RAMSTORAGE_HEADING": "Haupspeicher im Verhaltnis zu präsistentem Speicher", + "RAMSTORAGE_DESCRIPTION": "Das Verhältnis von Haupspeicher und prästistentem Speicher beinflusst die Leistung ganz allgemein. Ein Verältnis um 1:24 wird empfohlen. Wenn die Geschwindigkeit nicht so wichtig ist, kann auch darunter gegangen werden. Für eine schnelleres Cluster, kann ein engeres Verhältnis gewählt werden, aber astronomische Werte bringen natürlich nichts.", + "RAMSTORAGE_INPUT_1": "Jedes GiB RAM dient", + "RAMSTORAGE_INPUT_2": "GiB von präsistentem Speicher.", + "RAM_HEADING": "RAM", + "RAM_DESCRIPTION_1": "Wie viel Hauptspeicher haben die Rechner auf welchen die Knoten laufen sollen? Es wird empfohlen die Hälfte des Haupspeichers der virtuellen Java Maschiene zur Verfügung zu stellen und den Rest dem Betriebsystem. 64GiB an Haupspeicher sind ein guter Richtwert.", + "RAM_DESCRIPTION_2": "Byte", + "RAM_DESCRIPTION_3": "Wenn mehr als 32GiB der JVM als Heap zugewiesen werden, ist es gut möglich, dass es Probleme bei der Garbage Collection gibt.", + "RAID_HEADING": "RAID?", + "RAID_DESCRIPTION": "CrateDB ist verteilt und selbst-heilend, Replikate der Daten werden im Cluster verteilt, wenn sie eingeschaltet sind. Das ist empfehlenswert, denn es macht RAID1 überflüssig und ermöglicht sogar den sicheren Einsatz von RAID0.", + "USECASE_HEADING": "Anwendungsfall", + "DATA_HEADING": "Daten", + "DATA_DESCRIPTION_1": "Hierum dreht es sich bei crate. Du hast vermutlich schon eine andere Datenbankt in Verwendung gehabt und weißt vermutlich um viel Daten es geht.", + "DATA_DESCRIPTION_2": "Wie viel wird eingefügt?", + "DATA_DESCRIPTION_3": "Byte pro", + "DATA_DESCRIPTION_4": "Und wie lange sollen sie gespeichert werden?", + "DATA_DESCRIPTION_5": "Weil crate horizontal skallierbar ist, können auch immer später noch weitere Knoten hinzugefügt werden, um die Daten länger auf zu bewahren.", + "DATA_DESCRIPTION_6": "ODER, wenn nicht bekannt ist, wann wie viel Daten eingefügt werden, kannst du auch einfach eine erwartete Tabellen Größe angeben.", + "DATA_DESCRIPTION_7": "Byte", + "PARTITION_HEADING": "Partitionierung", + "PARTITION_DESCRIPTION_1": "crate organisirt die Daten in logische Einheiten, typischerweise nach Zeit. Wenn ein guter Zeitraum gewählt ist, kann das die Leistung drastisch erhöhen.", + "PARTITION_DESCRIPTION_2": "Wähle eine Größeneinheit, in der die Daten später verwendet werden. Wenn du keine Ahnung hast, dann ist ein Monat eine gute Empfehlung.", + "PARTITION_DESCRIPTION_3": "Du hast 'erwartete Tabellen Größe' ausgewählt, in dem Fall musst du die Anzahl der Partitionen manuell festlegen.", + "REDUNDANCY_HEADING": "Redundanz", + "REDUNDANCY_DESCRIPTION": "Wie gesagt verteilt crate Replikate über das ganze Cluster, ein Replikat sollte speicherplatzmäßig leistbar sein. Um Redundanz und um Datenintegrität zu gewährleisten. Keine Replikate zu haben ist naturlich am sparsamsten was den Speicher angeht, ist aber nicht empfehlenswert. Mehrere Replikate können die Abfragegeschwindigkeit signiffikant erhöhen, denn die Abfrage kann simultan auf mehreren Kopien des selben Datensatzes laufen. Der Nachteil ist natürlich der Speicherbedarf und es beinträchtigt sogar die Schreibgeschwindigkeit ein Bisschen. Demnach einst ein Replikat der normale Weg, wenn aber die Lesegeschwindigkeit wichtig ist kann man auch 3 oder 4 nehmen. Abgesehen davon, mehr Replikate als Knoten machen keinen Sinn. Sie können nirgendwo hin.", + "CALCULATION_HEADING": "Berechnung", + "CALCULATION_DESCRIPTION": "Mit den bereitgestellten Informationen, kommt dieses kleine Skript auf folgende Empfehlung:", + "NODES_HEADING": "Knoten", + "NODES_DESCRIPTION_1": "Du solltest", + "NODES_DESCRIPTION_2": "Knoten mit jeweils", + "NODES_DESCRIPTION_3": "Haupspeicher und", + "NODES_DESCRIPTION_4": "an präsistentem Speicher einsetzen.", + "SHARDS_HEADING": "Shards", + "SHARDS_DESCRIPTION_1": "Die Partitionen sollten jeweils in", + "SHARDS_DESCRIPTION_2": "Shards unterteilt werden.", + "HOUR": "Stunde", + "DAY": "Tag", + "WEEK": "Woche", + "MONTH": "Monat", + "YEAR": "Jahr" } } diff --git a/app/plugins/tutorial/static/i18n/en.json b/app/plugins/tutorial/static/i18n/en.json index 400133c4..c78c90df 100644 --- a/app/plugins/tutorial/static/i18n/en.json +++ b/app/plugins/tutorial/static/i18n/en.json @@ -4,8 +4,8 @@ }, "TUTORIAL": { "TITLE": "Get started with sample data", - "PARAGRAPH_1": "Sometimes it helps just to test with something real. The Twitter importer lets you quickly test your new CrateDB instance with real data. Press the button, and watch your cluster quickly fill up in real time with live Tweets. You can always delete them later.", - "PARAGRAPH_2": "After authentication Twitter allows you to consume a small fraction of their public stream, hence the slow import. In a production environment you can index hundreds of thousands of records per second.", + "PARAGRAPH_1": "Sometimes it helps to test with something real. The Twitter importer let's you quickly test your new CrateDB instance with real data. Press the button, and watch your cluster fill up in real time with live Tweets. You can always delete them later.", + "PARAGRAPH_2": "After the authentication, Twitter allows you to consume a small fraction of their public stream, hence the slow import. In a production environment you can import hundreds of thousands of records per second.", "INSTRUCTION_1": "Your imported tweets will be stored in a table called {tweets}.", "TWEETS": "tweets", "INSTRUCTION_2": "Click {tables} on the left to see the table and its stats. You should see its records increase as the tweets import.", @@ -16,5 +16,55 @@ "IMPORTING_TWEETS": "Importing tweets.", "TWEETS_IMPORTED": "tweets imported", "STOP_IMPORTING_TWEETS": "Enough! Stop importing tweets" + }, + "CALCULATOR": { + "TITLE": "Sharding calculator", + "DESCRIPTION": "A simple tool to calculate some generic sharding recommendation for a crate table.", + "IMPORT_HEADING": "Read data from existing table", + "HARDWARE_HEADING": "Hardware", + "CPUS_HEADING": "CPUs", + "CPUS_DESCRIPTION": "How many cores do the machines have you are going to run your nodes on?", + "STORAGE_HEADING": "Storage", + "STORAGE_DESCRIPTION": "You are using SSDs instead of HDDs? Good.", + "RAMSTORAGE_HEADING": "RAM/Storage", + "RAMSTORAGE_DESCRIPTION": "The proportion of RAM and storage generally influences the performance. A ratio around 1:24 is recommended, if you do not care for speed that much, you can go a little below that, if you do, you can go higher, but astronomical are not really improve performance anymore.", + "RAMSTORAGE_INPUT_1": "Every GiB of RAM serves", + "RAMSTORAGE_INPUT_2": "GiB of storage.", + "RAM_HEADING": "RAM", + "RAM_DESCRIPTION_1": "How much RAM do the machines have you are going to run your nodes on? It is good practice to have half of it assigned as heap to the JVM and the rest for the OS. 64GiB are a good guide value.", + "RAM_DESCRIPTION_2": "Byte", + "RAM_DESCRIPTION_3": "Having more than 32GiB of Heap for the JVM is not very beneficial, it will run into garbage collection issues.", + "RAID_HEADING": "RAID?", + "RAID_DESCRIPTION": "CrateDB is distributed and self healing, replicas of the data are distributed among the cluster, if they are enabled. This is probably something you want, because it makes something like RAID1 superfluous and even enables the use of RAID0 in a safe way.", + "USECASE_HEADING": "Use-Case", + "DATA_HEADING": "Data", + "DATA_DESCRIPTION_1": "So, this is what crate is about. You have probably been using something else before, so you should generally know how much data there is.", + "DATA_DESCRIPTION_2": "How much data is being inserted?", + "DATA_DESCRIPTION_3": "Byte per", + "DATA_DESCRIPTION_4": "And for how long is it supposed to be stored?", + "DATA_DESCRIPTION_5": "Since crate is horizontally scalable you can later on decide that the data is supposed to be stored for longer, by adding nodes.", + "DATA_DESCRIPTION_6": "OR, if it’s really not known yet, simply specify some expected table size.", + "DATA_DESCRIPTION_7": "Byte", + "PARTITION_HEADING": "Partitioning", + "PARTITION_DESCRIPTION_1": "Crate organizes data by itself into logical units, usually by time. If setting a good time frame for partitioning is set, can greatly enhance the performance.", + "PARTITION_DESCRIPTION_2": "Choose a batch size in which the data will be used later. If you have no idea, one month is a good suggestion.", + "PARTITION_DESCRIPTION_3": "You have expected table size selected, in this case you need to specify the amount of partitions manually.", + "REDUNDANCY_HEADING": "Redundancy", + "REDUNDANCY_DESCRIPTION": "As said crate distributes replicas of the data among its network, one replica should be affordable storage wise, for redundancy and data integrity. Having no replicas is of course easy on storage but not advisable. By having more replicas the query speed can be enhanced significantly, because one query then can run on several copies of the same data simultaneously. Sadly that is very storage intense and does affect the write speed a little. So one replica is the normal way to go if you are very into query speed you can go with 3-4. Aside from that, having more replicas than nodes never makes sense. They have nowhere to go.", + "CALCULATION_HEADING": "Calculation", + "CALCULATION_DESCRIPTION": "Based on the provided data, this little script came up with a recommendation.", + "NODES_HEADING": "Nodes", + "NODES_DESCRIPTION_1": "You should use", + "NODES_DESCRIPTION_2": "nodes with", + "NODES_DESCRIPTION_3": "of RAM and", + "NODES_DESCRIPTION_4": "of storage each.", + "SHARDS_HEADING": "Shards", + "SHARDS_DESCRIPTION_1": "The partitions should be divided into", + "SHARDS_DESCRIPTION_2": "shards each.", + "HOUR": "hour", + "DAY": "day", + "WEEK": "week", + "MONTH": "month", + "YEAR": "year" } } diff --git a/app/plugins/tutorial/static/icons/sharding-calculator-logo.png b/app/plugins/tutorial/static/icons/sharding-calculator-logo.png new file mode 100644 index 00000000..476aed45 Binary files /dev/null and b/app/plugins/tutorial/static/icons/sharding-calculator-logo.png differ diff --git a/app/plugins/tutorial/tutorial.html b/app/plugins/tutorial/tutorial.html old mode 100644 new mode 100755 index 4f994db9..f0eaf249 --- a/app/plugins/tutorial/tutorial.html +++ b/app/plugins/tutorial/tutorial.html @@ -77,6 +77,14 @@

{{:: 'HELP.ENTERPRISE_TITLE' | translate}}
{{:: 'HELP.ENTERPRISE_MSG' | translate }}
+ + +
+
+

{{:: 'HELP.CALCULATOR_TITLE' | translate}}

+
{{:: 'HELP.CALCULATOR_MSG' | translate}}
+
+ diff --git a/app/scripts/filter/numbers.js b/app/scripts/filter/numbers.js index 12fba473..8a357a44 100644 --- a/app/scripts/filter/numbers.js +++ b/app/scripts/filter/numbers.js @@ -27,7 +27,7 @@ const filters_numbers = angular.module('filters_numbers', []) }; }) .filter('bytes', function($sce) { - var units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']; + var units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; return function(bytes, precision) { if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) { return '-'; diff --git a/app/static/i18n/de.json b/app/static/i18n/de.json index c9836eda..a6c39f51 100644 --- a/app/static/i18n/de.json +++ b/app/static/i18n/de.json @@ -142,6 +142,8 @@ "DOCUMENTATION_TITLE": "CrateDB Dokumentation", "DOCUMENTATION_MSG": "Die CrateDB Online Dokumentation.", "ENTERPRISE_TITLE": "Enterprise Support", - "ENTERPRISE_MSG": "Mehr über den Crate.io Enterprise Support." + "ENTERPRISE_MSG": "Mehr über den Crate.io Enterprise Support.", + "CALCULATOR_TITLE": "Shard Rechner", + "CALCULATOR_MSG": "Interaktives Werkzeug zur Berechnung, von Shard Größen." } } diff --git a/app/static/i18n/en.json b/app/static/i18n/en.json index 8fe47465..e8d47d4e 100644 --- a/app/static/i18n/en.json +++ b/app/static/i18n/en.json @@ -142,6 +142,8 @@ "DOCUMENTATION_TITLE": "CrateDB Documentation", "DOCUMENTATION_MSG": "Read the CrateDB documentation.", "ENTERPRISE_TITLE": "Enterprise Support", - "ENTERPRISE_MSG": "For enterprise support, please check our pricing page." + "ENTERPRISE_MSG": "For enterprise support, please check our pricing page.", + "CALCULATOR_TITLE": "Sharding Calculator", + "CALCULATOR_MSG": "Get some help on how to split your table into shards." } }